pym/portage/repository/config.py

   1 # Copyright 2010-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 import io
   7 import logging
   8 import warnings
   9 import sys
  10 import re
  11
  12 try:
  13         from configparser import Error as ConfigParserError
  14         if sys.hexversion >= 0x3020000:
  15                 from configparser import ConfigParser as SafeConfigParser
  16         else:
  17                 from configparser import SafeConfigParser
  18 except ImportError:
  19         from ConfigParser import SafeConfigParser, Error as ConfigParserError
  20 import portage
  21 from portage import eclass_cache, os
  22 from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH,
  23         PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH)
  24 from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level
  25 from portage.env.loaders import KeyValuePairFileLoader
  26 from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split,
  27         stack_lists, writemsg, writemsg_level, _recursive_file_list)
  28 from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess
  29 from portage.localization import _
  30 from portage import _unicode_decode
  31 from portage import _unicode_encode
  32 from portage import _encodings
  33 from portage import manifest
  34
  35 if sys.hexversion >= 0x3000000:
  36         basestring = str
  37
  38 # Characters prohibited by repoman's file.name check.
  39 _invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]')
  40
  41 _valid_profile_formats = frozenset(
  42         ['pms', 'portage-1', 'portage-2'])
  43
  44 _portage1_profiles_allow_directories = frozenset(
  45         ["portage-1-compat", "portage-1", 'portage-2'])
  46
  47 _repo_name_sub_re = re.compile(r'[^\w-]')
  48
  49 def _gen_valid_repo(name):
  50         """
  51         Substitute hyphen in place of characters that don't conform to PMS 3.1.5,
  52         and strip hyphen from left side if necessary. This returns None if the
  53         given name contains no valid characters.
  54         """
  55         name = _repo_name_sub_re.sub(' ', name.strip())
  56         name = '-'.join(name.split())
  57         name = name.lstrip('-')
  58         if not name:
  59                 name = None
  60         return name
  61
  62 def _find_invalid_path_char(path, pos=0, endpos=None):
  63         """
  64         Returns the position of the first invalid character found in basename,
  65         or -1 if no invalid characters are found.
  66         """
  67         if endpos is None:
  68                 endpos = len(path)
  69
  70         m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos)
  71         if m is not None:
  72                 return m.start()
  73
  74         return -1
  75
  76 class RepoConfig(object):
  77         """Stores config of one repository"""
  78
  79         __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual',
  80                 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi',
  81                 'eclass_db', 'eclass_locations', 'eclass_overrides',
  82                 'find_invalid_path_char', 'force', 'format', 'local_config', 'location',
  83                 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
  84                 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
  85                 'profile_formats', 'sign_commit', 'sign_manifest', 'sync_cvs_repo',
  86                 'sync_type', 'sync_uri', 'thin_manifest', 'update_changelog',
  87                 'user_location', '_eapis_banned', '_eapis_deprecated')
  88
  89         def __init__(self, name, repo_opts, local_config=True):
  90                 """Build a RepoConfig with options in repo_opts
  91                    Try to read repo_name in repository location, but if
  92                    it is not found use variable name as repository name"""
  93
  94                 force = repo_opts.get('force')
  95                 if force is not None:
  96                         force = tuple(force.split())
  97                 self.force = force
  98                 if force is None:
  99                         force = ()
 100
 101                 self.local_config = local_config
 102
 103                 if local_config or 'aliases' in force:
 104                         aliases = repo_opts.get('aliases')
 105                         if aliases is not None:
 106                                 aliases = tuple(aliases.split())
 107                 else:
 108                         aliases = None
 109
 110                 self.aliases = aliases
 111
 112                 if local_config or 'eclass-overrides' in force:
 113                         eclass_overrides = repo_opts.get('eclass-overrides')
 114                         if eclass_overrides is not None:
 115                                 eclass_overrides = tuple(eclass_overrides.split())
 116                 else:
 117                         eclass_overrides = None
 118
 119                 self.eclass_overrides = eclass_overrides
 120                 # Eclass databases and locations are computed later.
 121                 self.eclass_db = None
 122                 self.eclass_locations = None
 123
 124                 if local_config or 'masters' in force:
 125                         # Masters from repos.conf override layout.conf.
 126                         masters = repo_opts.get('masters')
 127                         if masters is not None:
 128                                 masters = tuple(masters.split())
 129                 else:
 130                         masters = None
 131
 132                 self.masters = masters
 133
 134                 #The main-repo key makes only sense for the 'DEFAULT' section.
 135                 self.main_repo = repo_opts.get('main-repo')
 136
 137                 priority = repo_opts.get('priority')
 138                 if priority is not None:
 139                         try:
 140                                 priority = int(priority)
 141                         except ValueError:
 142                                 priority = None
 143                 self.priority = priority
 144
 145                 sync_cvs_repo = repo_opts.get('sync-cvs-repo')
 146                 if sync_cvs_repo is not None:
 147                         sync_cvs_repo = sync_cvs_repo.strip()
 148                 self.sync_cvs_repo = sync_cvs_repo or None
 149
 150                 sync_type = repo_opts.get('sync-type')
 151                 if sync_type is not None:
 152                         sync_type = sync_type.strip()
 153                 self.sync_type = sync_type or None
 154
 155                 sync_uri = repo_opts.get('sync-uri')
 156                 if sync_uri is not None:
 157                         sync_uri = sync_uri.strip()
 158                 self.sync_uri = sync_uri or None
 159
 160                 # Not implemented.
 161                 format = repo_opts.get('format')
 162                 if format is not None:
 163                         format = format.strip()
 164                 self.format = format
 165
 166                 location = repo_opts.get('location')
 167                 self.user_location = location
 168                 if location is not None and location.strip():
 169                         if os.path.isdir(location) or portage._sync_disabled_warnings:
 170                                 location = os.path.realpath(location)
 171                 else:
 172                         location = None
 173                 self.location = location
 174
 175                 eapi = None
 176                 missing = True
 177                 self.name = name
 178                 if self.location is not None:
 179                         if os.path.isdir(location):
 180                                 eapi = read_corresponding_eapi_file(os.path.join(self.location, REPO_NAME_LOC))
 181                                 self.name, missing = self._read_valid_repo_name(self.location)
 182                         else:
 183                                 missing = not portage._sync_disabled_warnings
 184                 elif name == "DEFAULT":
 185                         missing = False
 186
 187                 self.eapi = eapi
 188                 self.missing_repo_name = missing
 189                 # sign_commit is disabled by default, since it requires Git >=1.7.9,
 190                 # and key_id configured by `git config user.signingkey key_id`
 191                 self.sign_commit = False
 192                 self.sign_manifest = True
 193                 self.thin_manifest = False
 194                 self.allow_missing_manifest = False
 195                 self.allow_provide_virtual = False
 196                 self.create_manifest = True
 197                 self.disable_manifest = False
 198                 self.manifest_hashes = None
 199                 self.update_changelog = False
 200                 self.cache_formats = None
 201                 self.portage1_profiles = True
 202                 self.portage1_profiles_compat = False
 203                 self.find_invalid_path_char = _find_invalid_path_char
 204
 205                 # Parse layout.conf.
 206                 if self.location:
 207                         layout_data = parse_layout_conf(self.location, self.name)[0]
 208
 209                         # layout.conf masters may be overridden here if we have a masters
 210                         # setting from the user's repos.conf
 211                         if self.masters is None:
 212                                 self.masters = layout_data['masters']
 213
 214                         if (local_config or 'aliases' in force) and layout_data['aliases']:
 215                                 aliases = self.aliases
 216                                 if aliases is None:
 217                                         aliases = ()
 218                                 # repos.conf aliases come after layout.conf aliases, giving
 219                                 # them the ability to do incremental overrides
 220                                 self.aliases = layout_data['aliases'] + tuple(aliases)
 221
 222                         if layout_data['repo-name']:
 223                                 # allow layout.conf to override repository name
 224                                 # useful when having two copies of the same repo enabled
 225                                 # to avoid modifying profiles/repo_name in one of them
 226                                 self.name = layout_data['repo-name']
 227
 228                         for value in ('allow-missing-manifest',
 229                                 'allow-provide-virtual', 'cache-formats',
 230                                 'create-manifest', 'disable-manifest', 'manifest-hashes',
 231                                 'profile-formats',
 232                                 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'):
 233                                 setattr(self, value.lower().replace("-", "_"), layout_data[value])
 234
 235                         self.portage1_profiles = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \
 236                                 any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats'])
 237                         self.portage1_profiles_compat = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \
 238                                 layout_data['profile-formats'] == ('portage-1-compat',)
 239
 240                         self._eapis_banned = frozenset(layout_data['eapis-banned'])
 241                         self._eapis_deprecated = frozenset(layout_data['eapis-deprecated'])
 242
 243         def eapi_is_banned(self, eapi):
 244                 return eapi in self._eapis_banned
 245
 246         def eapi_is_deprecated(self, eapi):
 247                 return eapi in self._eapis_deprecated
 248
 249         def iter_pregenerated_caches(self, auxdbkeys, readonly=True, force=False):
 250                 """
 251                 Reads layout.conf cache-formats from left to right and yields cache
 252                 instances for each supported type that's found. If no cache-formats
 253                 are specified in layout.conf, 'pms' type is assumed if the
 254                 metadata/cache directory exists or force is True.
 255                 """
 256                 formats = self.cache_formats
 257                 if not formats:
 258                         if not force:
 259                                 return
 260                         # The default egencache format was 'pms' prior to portage-2.1.11.32
 261                         # (portage versions prior to portage-2.1.11.14 will NOT
 262                         # recognize md5-dict format unless it is explicitly listed in
 263                         # layout.conf).
 264                         formats = ('md5-dict',)
 265
 266                 for fmt in formats:
 267                         name = None
 268                         if fmt == 'pms':
 269                                 from portage.cache.metadata import database
 270                                 name = 'metadata/cache'
 271                         elif fmt == 'md5-dict':
 272                                 from portage.cache.flat_hash import md5_database as database
 273                                 name = 'metadata/md5-cache'
 274
 275                         if name is not None:
 276                                 yield database(self.location, name,
 277                                         auxdbkeys, readonly=readonly)
 278
 279         def get_pregenerated_cache(self, auxdbkeys, readonly=True, force=False):
 280                 """
 281                 Returns the first cache instance yielded from
 282                 iter_pregenerated_caches(), or None if no cache is available or none
 283                 of the available formats are supported.
 284                 """
 285                 return next(self.iter_pregenerated_caches(
 286                         auxdbkeys, readonly=readonly, force=force), None)
 287
 288         def load_manifest(self, *args, **kwds):
 289                 kwds['thin'] = self.thin_manifest
 290                 kwds['allow_missing'] = self.allow_missing_manifest
 291                 kwds['allow_create'] = self.create_manifest
 292                 kwds['hashes'] = self.manifest_hashes
 293                 if self.disable_manifest:
 294                         kwds['from_scratch'] = True
 295                 kwds['find_invalid_path_char'] = self.find_invalid_path_char
 296                 return manifest.Manifest(*args, **portage._native_kwargs(kwds))
 297
 298         def update(self, new_repo):
 299                 """Update repository with options in another RepoConfig"""
 300
 301                 keys = set(self.__slots__)
 302                 keys.discard("missing_repo_name")
 303                 for k in keys:
 304                         v = getattr(new_repo, k, None)
 305                         if v is not None:
 306                                 setattr(self, k, v)
 307
 308                 if new_repo.name is not None:
 309                         self.missing_repo_name = new_repo.missing_repo_name
 310
 311         @staticmethod
 312         def _read_valid_repo_name(repo_path):
 313                 name, missing = RepoConfig._read_repo_name(repo_path)
 314                 # We must ensure that the name conforms to PMS 3.1.5
 315                 # in order to avoid InvalidAtom exceptions when we
 316                 # use it to generate atoms.
 317                 name = _gen_valid_repo(name)
 318                 if not name:
 319                         # name only contains invalid characters
 320                         name = "x-" + os.path.basename(repo_path)
 321                         name = _gen_valid_repo(name)
 322                         # If basename only contains whitespace then the
 323                         # end result is name = 'x-'.
 324                 return name, missing
 325
 326         @staticmethod
 327         def _read_repo_name(repo_path):
 328                 """
 329                 Read repo_name from repo_path.
 330                 Returns repo_name, missing.
 331                 """
 332                 repo_name_path = os.path.join(repo_path, REPO_NAME_LOC)
 333                 f = None
 334                 try:
 335                         f = io.open(
 336                                 _unicode_encode(repo_name_path,
 337                                 encoding=_encodings['fs'], errors='strict'),
 338                                 mode='r', encoding=_encodings['repo.content'],
 339                                 errors='replace')
 340                         return f.readline().strip(), False
 341                 except EnvironmentError:
 342                         return "x-" + os.path.basename(repo_path), True
 343                 finally:
 344                         if f is not None:
 345                                 f.close()
 346
 347         def info_string(self):
 348                 """
 349                 Returns a formatted string containing informations about the repository.
 350                 Used by emerge --info.
 351                 """
 352                 indent = " " * 4
 353                 repo_msg = []
 354                 repo_msg.append(self.name)
 355                 if self.format:
 356                         repo_msg.append(indent + "format: " + self.format)
 357                 if self.user_location:
 358                         repo_msg.append(indent + "location: " + self.user_location)
 359                 if self.sync_cvs_repo:
 360                         repo_msg.append(indent + "sync-cvs-repo: " + self.sync_cvs_repo)
 361                 if self.sync_type:
 362                         repo_msg.append(indent + "sync-type: " + self.sync_type)
 363                 if self.sync_uri:
 364                         repo_msg.append(indent + "sync-uri: " + self.sync_uri)
 365                 if self.masters:
 366                         repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters))
 367                 if self.priority is not None:
 368                         repo_msg.append(indent + "priority: " + str(self.priority))
 369                 if self.aliases:
 370                         repo_msg.append(indent + "aliases: " + " ".join(self.aliases))
 371                 if self.eclass_overrides:
 372                         repo_msg.append(indent + "eclass-overrides: " + \
 373                                 " ".join(self.eclass_overrides))
 374                 repo_msg.append("")
 375                 return "\n".join(repo_msg)
 376
 377         def __repr__(self):
 378                 return "<portage.repository.config.RepoConfig(name=%r, location=%r)>" % (self.name, _unicode_decode(self.location))
 379
 380         def __str__(self):
 381                 d = {}
 382                 for k in self.__slots__:
 383                         d[k] = getattr(self, k, None)
 384                 return "%s" % (d,)
 385
 386         if sys.hexversion < 0x3000000:
 387
 388                 __unicode__ = __str__
 389
 390                 def __str__(self):
 391                         return _unicode_encode(self.__unicode__())
 392
 393 class RepoConfigLoader(object):
 394         """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf"""
 395
 396         @staticmethod
 397         def _add_repositories(portdir, portdir_overlay, prepos,
 398                 ignored_map, ignored_location_map, local_config, default_portdir):
 399                 """Add overlays in PORTDIR_OVERLAY as repositories"""
 400                 overlays = []
 401                 portdir_orig = None
 402                 if portdir:
 403                         portdir = normalize_path(portdir)
 404                         portdir_orig = portdir
 405                         overlays.append(portdir)
 406                 try:
 407                         port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
 408                 except ValueError as e:
 409                         #File "/usr/lib/python3.2/shlex.py", line 168, in read_token
 410                         #       raise ValueError("No closing quotation")
 411                         writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
 412                                 " %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
 413                         port_ov = []
 414                 overlays.extend(port_ov)
 415                 default_repo_opts = {}
 416                 if prepos['DEFAULT'].aliases is not None:
 417                         default_repo_opts['aliases'] = \
 418                                 ' '.join(prepos['DEFAULT'].aliases)
 419                 if prepos['DEFAULT'].eclass_overrides is not None:
 420                         default_repo_opts['eclass-overrides'] = \
 421                                 ' '.join(prepos['DEFAULT'].eclass_overrides)
 422                 if prepos['DEFAULT'].masters is not None:
 423                         default_repo_opts['masters'] = \
 424                                 ' '.join(prepos['DEFAULT'].masters)
 425
 426                 if overlays:
 427                         # We need a copy of the original repos.conf data, since we're
 428                         # going to modify the prepos dict and some of the RepoConfig
 429                         # objects that we put in prepos may have to be discarded if
 430                         # they get overridden by a repository with the same name but
 431                         # a different location. This is common with repoman, for example,
 432                         # when temporarily overriding an rsync repo with another copy
 433                         # of the same repo from CVS.
 434                         repos_conf = prepos.copy()
 435                         #overlay priority is negative because we want them to be looked before any other repo
 436                         base_priority = 0
 437                         for ov in overlays:
 438                                 if isdir_raise_eaccess(ov):
 439                                         repo_opts = default_repo_opts.copy()
 440                                         repo_opts['location'] = ov
 441                                         repo = RepoConfig(None, repo_opts, local_config=local_config)
 442                                         # repos_conf_opts contains options from repos.conf
 443                                         repos_conf_opts = repos_conf.get(repo.name)
 444                                         if repos_conf_opts is not None:
 445                                                 # Selectively copy only the attributes which
 446                                                 # repos.conf is allowed to override.
 447                                                 for k in ('aliases', 'eclass_overrides', 'force', 'masters',
 448                                                         'priority', 'sync_cvs_repo', 'sync_type', 'sync_uri'):
 449                                                         v = getattr(repos_conf_opts, k, None)
 450                                                         if v is not None:
 451                                                                 setattr(repo, k, v)
 452
 453                                         if repo.name in prepos:
 454                                                 old_location = prepos[repo.name].location
 455                                                 if old_location is not None and \
 456                                                         old_location != repo.location and \
 457                                                         not (old_location == default_portdir and
 458                                                         not exists_raise_eaccess(old_location)):
 459                                                         ignored_map.setdefault(repo.name, []).append(old_location)
 460                                                         ignored_location_map[old_location] = repo.name
 461                                                         if old_location == portdir:
 462                                                                 portdir = repo.user_location
 463
 464                                         if repo.priority is None:
 465                                                 if base_priority == 0 and ov == portdir_orig:
 466                                                         # If it's the original PORTDIR setting and it's not
 467                                                         # in PORTDIR_OVERLAY, then it will be assigned a
 468                                                         # special priority setting later.
 469                                                         pass
 470                                                 else:
 471                                                         repo.priority = base_priority
 472                                                         base_priority += 1
 473
 474                                         prepos[repo.name] = repo
 475                                 else:
 476
 477                                         if not portage._sync_disabled_warnings:
 478                                                 writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1)
 479
 480                 return portdir
 481
 482         @staticmethod
 483         def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir):
 484                 """Parse files in paths to load config"""
 485                 parser = SafeConfigParser()
 486
 487                 # use read_file/readfp in order to control decoding of unicode
 488                 try:
 489                         # Python >=3.2
 490                         read_file = parser.read_file
 491                         source_kwarg = 'source'
 492                 except AttributeError:
 493                         read_file = parser.readfp
 494                         source_kwarg = 'filename'
 495
 496                 recursive_paths = []
 497                 for p in paths:
 498                         if isinstance(p, basestring):
 499                                 recursive_paths.extend(_recursive_file_list(p))
 500                         else:
 501                                 recursive_paths.append(p)
 502
 503                 for p in recursive_paths:
 504                         if isinstance(p, basestring):
 505                                 f = None
 506                                 try:
 507                                         f = io.open(_unicode_encode(p,
 508                                                 encoding=_encodings['fs'], errors='strict'),
 509                                                 mode='r', encoding=_encodings['repo.content'],
 510                                                 errors='replace')
 511                                 except EnvironmentError:
 512                                         pass
 513                                 else:
 514                                         # The 'source' keyword argument is needed since otherwise
 515                                         # ConfigParser in Python <3.3.3 may throw a TypeError
 516                                         # because it assumes that f.name is a native string rather
 517                                         # than binary when constructing error messages.
 518                                         kwargs = {source_kwarg: p}
 519                                         read_file(f, **portage._native_kwargs(kwargs))
 520                                 finally:
 521                                         if f is not None:
 522                                                 f.close()
 523                         elif isinstance(p, io.StringIO):
 524                                 kwargs = {source_kwarg: "<io.StringIO>"}
 525                                 read_file(p, **portage._native_kwargs(kwargs))
 526                         else:
 527                                 raise TypeError("Unsupported type %r of element %r of 'paths' argument" % (type(p), p))
 528
 529                 prepos['DEFAULT'] = RepoConfig("DEFAULT",
 530                         parser.defaults(), local_config=local_config)
 531
 532                 for sname in parser.sections():
 533                         optdict = {}
 534                         for oname in parser.options(sname):
 535                                 optdict[oname] = parser.get(sname, oname)
 536
 537                         repo = RepoConfig(sname, optdict, local_config=local_config)
 538
 539                         if repo.sync_type is not None and repo.sync_uri is None:
 540                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute, but is missing sync-uri attribute") %
 541                                         sname, level=logging.ERROR, noiselevel=-1)
 542                                 continue
 543
 544                         if repo.sync_uri is not None and repo.sync_type is None:
 545                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-uri attribute, but is missing sync-type attribute") %
 546                                         sname, level=logging.ERROR, noiselevel=-1)
 547                                 continue
 548
 549                         if repo.sync_type not in (None, "cvs", "git", "rsync"):
 550                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute set to unsupported value: '%s'") %
 551                                         (sname, repo.sync_type), level=logging.ERROR, noiselevel=-1)
 552                                 continue
 553
 554                         if repo.sync_type == "cvs" and repo.sync_cvs_repo is None:
 555                                 writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute") %
 556                                         sname, level=logging.ERROR, noiselevel=-1)
 557                                 continue
 558
 559                         # For backward compatibility with locations set via PORTDIR and
 560                         # PORTDIR_OVERLAY, delay validation of the location and repo.name
 561                         # until after PORTDIR and PORTDIR_OVERLAY have been processed.
 562                         prepos[sname] = repo
 563
 564         def __init__(self, paths, settings):
 565                 """Load config from files in paths"""
 566
 567                 prepos = {}
 568                 location_map = {}
 569                 treemap = {}
 570                 ignored_map = {}
 571                 ignored_location_map = {}
 572
 573                 if "PORTAGE_REPOSITORIES" in settings:
 574                         portdir = ""
 575                         portdir_overlay = ""
 576                 else:
 577                         portdir = settings.get("PORTDIR", "")
 578                         portdir_overlay = settings.get("PORTDIR_OVERLAY", "")
 579
 580                 try:
 581                         self._parse(paths, prepos, ignored_map,
 582                                 ignored_location_map, settings.local_config,
 583                                 portdir)
 584                 except ConfigParserError as e:
 585                         writemsg(
 586                                 _("!!! Error while reading repo config file: %s\n") % e,
 587                                 noiselevel=-1)
 588                         # The configparser state is unreliable (prone to quirky
 589                         # exceptions) after it has thrown an error, so use empty
 590                         # config and try to fall back to PORTDIR{,_OVERLAY}.
 591                         prepos.clear()
 592                         prepos['DEFAULT'] = RepoConfig('DEFAULT',
 593                                 {}, local_config=settings.local_config)
 594                         location_map.clear()
 595                         treemap.clear()
 596                         ignored_map.clear()
 597                         ignored_location_map.clear()
 598
 599                 default_portdir = os.path.join(os.sep,
 600                         settings['EPREFIX'].lstrip(os.sep), 'usr', 'portage')
 601
 602                 # If PORTDIR_OVERLAY contains a repo with the same repo_name as
 603                 # PORTDIR, then PORTDIR is overridden.
 604                 portdir = self._add_repositories(portdir, portdir_overlay, prepos,
 605                         ignored_map, ignored_location_map, settings.local_config,
 606                         default_portdir)
 607                 if portdir and portdir.strip():
 608                         portdir = os.path.realpath(portdir)
 609
 610                 ignored_repos = tuple((repo_name, tuple(paths)) \
 611                         for repo_name, paths in ignored_map.items())
 612
 613                 self.missing_repo_names = frozenset(repo.location
 614                         for repo in prepos.values()
 615                         if repo.location is not None and repo.missing_repo_name)
 616
 617                 # Do this before expanding aliases, so that location_map and
 618                 # treemap consistently map unaliased names whenever available.
 619                 for repo_name, repo in list(prepos.items()):
 620                         if repo.location is None:
 621                                 if repo_name != 'DEFAULT':
 622                                         # Skip this warning for repoman (bug #474578).
 623                                         if settings.local_config and paths:
 624                                                 writemsg_level(_("Location undefined for " \
 625                                                         "repository '%s' referenced in '%s'\n") % \
 626                                                         (repo.name, (paths if len(paths) > 1 else paths[0])),
 627                                                         level=logging.ERROR, noiselevel=-1)
 628                                         del prepos[repo_name]
 629                         else:
 630                                 if repo.name != repo_name and not portage._sync_disabled_warnings:
 631                                         writemsg_level("!!! %s\n" % _("Section name '%s' set in repos.conf differs from name '%s' set inside repository") %
 632                                                 (repo_name, repo.name), level=logging.ERROR, noiselevel=-1)
 633                                         del prepos[repo_name]
 634                                         continue
 635
 636                                 if repo.location and not exists_raise_eaccess(repo.location) and not portage._sync_disabled_warnings:
 637                                         writemsg_level("!!! %s\n" % _("Repository '%s' has location attribute set to nonexistent directory: '%s'") %
 638                                                 (repo_name, repo.location), level=logging.ERROR, noiselevel=-1)
 639                                         del prepos[repo_name]
 640                                         continue
 641
 642                                 location_map[repo.location] = repo_name
 643                                 treemap[repo_name] = repo.location
 644
 645                 # Add alias mappings, but never replace unaliased mappings.
 646                 for repo_name, repo in list(prepos.items()):
 647                         names = set()
 648                         names.add(repo_name)
 649                         if repo.aliases:
 650                                 aliases = stack_lists([repo.aliases], incremental=True)
 651                                 names.update(aliases)
 652
 653                         for name in names:
 654                                 if name in prepos and prepos[name].location is not None:
 655                                         if name == repo_name:
 656                                                 # unaliased names already handled earlier
 657                                                 continue
 658                                         writemsg_level(_("!!! Repository name or alias '%s', " + \
 659                                                 "defined for repository '%s', overrides " + \
 660                                                 "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1)
 661                                         # Never replace an unaliased mapping with
 662                                         # an aliased mapping.
 663                                         continue
 664                                 prepos[name] = repo
 665                                 if repo.location is not None:
 666                                         if repo.location not in location_map:
 667                                                 # Never replace an unaliased mapping with
 668                                                 # an aliased mapping.
 669                                                 location_map[repo.location] = name
 670                                         treemap[name] = repo.location
 671
 672                 main_repo = prepos['DEFAULT'].main_repo
 673                 if main_repo is None or main_repo not in prepos:
 674                         #setting main_repo if it was not set in repos.conf
 675                         main_repo = location_map.get(portdir)
 676                         if main_repo is not None:
 677                                 prepos['DEFAULT'].main_repo = main_repo
 678                         else:
 679                                 prepos['DEFAULT'].main_repo = None
 680                                 if not portage._sync_disabled_warnings:
 681                                         writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty.\n"), noiselevel=-1)
 682
 683                 if main_repo is not None and prepos[main_repo].priority is None:
 684                         # This happens if main-repo has been set in repos.conf.
 685                         prepos[main_repo].priority = -1000
 686
 687                 # Include repo.name in sort key, for predictable sorting
 688                 # even when priorities are equal.
 689                 prepos_order = sorted(prepos.items(),
 690                         key=lambda r:(r[1].priority or 0, r[1].name))
 691
 692                 # filter duplicates from aliases, by only including
 693                 # items where repo.name == key
 694                 prepos_order = [repo.name for (key, repo) in prepos_order
 695                         if repo.name == key and key != 'DEFAULT' and
 696                         repo.location is not None]
 697
 698                 self.prepos = prepos
 699                 self.prepos_order = prepos_order
 700                 self.ignored_repos = ignored_repos
 701                 self.location_map = location_map
 702                 self.treemap = treemap
 703                 self._prepos_changed = True
 704                 self._repo_location_list = []
 705
 706                 #The 'masters' key currently contains repo names. Replace them with the matching RepoConfig.
 707                 for repo_name, repo in prepos.items():
 708                         if repo_name == "DEFAULT":
 709                                 continue
 710                         if repo.masters is None:
 711                                 if self.mainRepo() and repo_name != self.mainRepo().name:
 712                                         repo.masters = self.mainRepo(),
 713                                 else:
 714                                         repo.masters = ()
 715                         else:
 716                                 if repo.masters and isinstance(repo.masters[0], RepoConfig):
 717                                         # This one has already been processed
 718                                         # because it has an alias.
 719                                         continue
 720                                 master_repos = []
 721                                 for master_name in repo.masters:
 722                                         if master_name not in prepos:
 723                                                 layout_filename = os.path.join(repo.user_location,
 724                                                         "metadata", "layout.conf")
 725                                                 writemsg_level(_("Unavailable repository '%s' " \
 726                                                         "referenced by masters entry in '%s'\n") % \
 727                                                         (master_name, layout_filename),
 728                                                         level=logging.ERROR, noiselevel=-1)
 729                                         else:
 730                                                 master_repos.append(prepos[master_name])
 731                                 repo.masters = tuple(master_repos)
 732
 733                 #The 'eclass_overrides' key currently contains repo names. Replace them with the matching repo paths.
 734                 for repo_name, repo in prepos.items():
 735                         if repo_name == "DEFAULT":
 736                                 continue
 737
 738                         eclass_locations = []
 739                         eclass_locations.extend(master_repo.location for master_repo in repo.masters)
 740                         # Only append the current repo to eclass_locations if it's not
 741                         # there already. This allows masters to have more control over
 742                         # eclass override order, which may be useful for scenarios in
 743                         # which there is a plan to migrate eclasses to a master repo.
 744                         if repo.location not in eclass_locations:
 745                                 eclass_locations.append(repo.location)
 746
 747                         if repo.eclass_overrides:
 748                                 for other_repo_name in repo.eclass_overrides:
 749                                         if other_repo_name in self.treemap:
 750                                                 eclass_locations.append(self.get_location_for_name(other_repo_name))
 751                                         else:
 752                                                 writemsg_level(_("Unavailable repository '%s' " \
 753                                                         "referenced by eclass-overrides entry for " \
 754                                                         "'%s'\n") % (other_repo_name, repo_name), \
 755                                                         level=logging.ERROR, noiselevel=-1)
 756                         repo.eclass_locations = tuple(eclass_locations)
 757
 758                 eclass_dbs = {}
 759                 for repo_name, repo in prepos.items():
 760                         if repo_name == "DEFAULT":
 761                                 continue
 762
 763                         eclass_db = None
 764                         for eclass_location in repo.eclass_locations:
 765                                 tree_db = eclass_dbs.get(eclass_location)
 766                                 if tree_db is None:
 767                                         tree_db = eclass_cache.cache(eclass_location)
 768                                         eclass_dbs[eclass_location] = tree_db
 769                                 if eclass_db is None:
 770                                         eclass_db = tree_db.copy()
 771                                 else:
 772                                         eclass_db.append(tree_db)
 773                         repo.eclass_db = eclass_db
 774
 775                 for repo_name, repo in prepos.items():
 776                         if repo_name == "DEFAULT":
 777                                 continue
 778
 779                         if parse_layout_conf(repo.location, repo_name)[0]["masters"] is None and self.mainRepo() and \
 780                                 repo_name != self.mainRepo().name and not portage._sync_disabled_warnings:
 781                                 writemsg_level("!!! %s\n" % _("Repository '%s' is missing masters attribute in '%s'") %
 782                                         (repo_name, os.path.join(repo.location, "metadata", "layout.conf")) +
 783                                         "!!! %s\n" % _("Set 'masters = %s' in this file for future compatibility") %
 784                                         self.mainRepo().name, level=logging.WARNING, noiselevel=-1)
 785
 786                 self._prepos_changed = True
 787                 self._repo_location_list = []
 788
 789                 self._check_locations()
 790
 791         def repoLocationList(self):
 792                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
 793                 if self._prepos_changed:
 794                         _repo_location_list = []
 795                         for repo in self.prepos_order:
 796                                 if self.prepos[repo].location is not None:
 797                                         _repo_location_list.append(self.prepos[repo].location)
 798                         self._repo_location_list = tuple(_repo_location_list)
 799
 800                         self._prepos_changed = False
 801                 return self._repo_location_list
 802
 803         def repoUserLocationList(self):
 804                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
 805                 user_location_list = []
 806                 for repo in self.prepos_order:
 807                         if self.prepos[repo].location is not None:
 808                                 user_location_list.append(self.prepos[repo].user_location)
 809                 return tuple(user_location_list)
 810
 811         def mainRepoLocation(self):
 812                 """Returns the location of main repo"""
 813                 main_repo = self.prepos['DEFAULT'].main_repo
 814                 if main_repo is not None and main_repo in self.prepos:
 815                         return self.prepos[main_repo].location
 816                 else:
 817                         return ''
 818
 819         def mainRepo(self):
 820                 """Returns the main repo"""
 821                 main_repo = self.prepos['DEFAULT'].main_repo
 822                 if main_repo is None:
 823                         return None
 824                 return self.prepos[main_repo]
 825
 826         def _check_locations(self):
 827                 """Check if repositories location are correct and show a warning message if not"""
 828                 for (name, r) in self.prepos.items():
 829                         if name != 'DEFAULT':
 830                                 if r.location is None:
 831                                         writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1)
 832                                 else:
 833                                         if not isdir_raise_eaccess(r.location) and not portage._sync_disabled_warnings:
 834                                                 self.prepos_order.remove(name)
 835                                                 writemsg(_("!!! Invalid Repository Location"
 836                                                         " (not a dir): '%s'\n") % r.location, noiselevel=-1)
 837
 838         def repos_with_profiles(self):
 839                 for repo_name in self.prepos_order:
 840                         repo = self.prepos[repo_name]
 841                         if repo.format != "unavailable":
 842                                 yield repo
 843
 844         def get_name_for_location(self, location):
 845                 return self.location_map[location]
 846
 847         def get_location_for_name(self, repo_name):
 848                 if repo_name is None:
 849                         # This simplifies code in places where
 850                         # we want to be able to pass in Atom.repo
 851                         # even if it is None.
 852                         return None
 853                 return self.treemap[repo_name]
 854
 855         def get_repo_for_location(self, location):
 856                 return self.prepos[self.get_name_for_location(location)]
 857
 858         def __setitem__(self, repo_name, repo):
 859                 # self.prepos[repo_name] = repo
 860                 raise NotImplementedError
 861
 862         def __getitem__(self, repo_name):
 863                 return self.prepos[repo_name]
 864
 865         def __delitem__(self, repo_name):
 866                 location = self.prepos[repo_name].location
 867                 del self.prepos[repo_name]
 868                 if repo_name in self.prepos_order:
 869                         self.prepos_order.remove(repo_name)
 870                 for k, v in self.location_map.copy().items():
 871                         if v == repo_name:
 872                                 del self.location_map[k]
 873                 if repo_name in self.treemap:
 874                         del self.treemap[repo_name]
 875                 self._repo_location_list = tuple(x for x in self._repo_location_list if x != location)
 876
 877         def __iter__(self):
 878                 for repo_name in self.prepos_order:
 879                         yield self.prepos[repo_name]
 880
 881         def __contains__(self, repo_name):
 882                 return repo_name in self.prepos
 883
 884         def config_string(self):
 885                 str_or_int_keys = ("format", "location", "main_repo", "priority", "sync_cvs_repo", "sync_type", "sync_uri")
 886                 str_tuple_keys = ("aliases", "eclass_overrides", "force")
 887                 repo_config_tuple_keys = ("masters",)
 888                 keys = str_or_int_keys + str_tuple_keys + repo_config_tuple_keys
 889                 config_string = ""
 890                 for repo_name, repo in sorted(self.prepos.items()):
 891                         config_string += "\n[%s]\n" % repo_name
 892                         for key in sorted(keys):
 893                                 if key == "main_repo" and repo_name != "DEFAULT":
 894                                         continue
 895                                 if getattr(repo, key) is not None:
 896                                         if key in str_or_int_keys:
 897                                                 config_string += "%s = %s\n" % (key.replace("_", "-"), getattr(repo, key))
 898                                         elif key in str_tuple_keys:
 899                                                 config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(getattr(repo, key)))
 900                                         elif key in repo_config_tuple_keys:
 901                                                 config_string += "%s = %s\n" % (key.replace("_", "-"), " ".join(x.name for x in getattr(repo, key)))
 902                 return config_string.lstrip("\n")
 903
 904 def load_repository_config(settings, extra_files=None):
 905         repoconfigpaths = []
 906         if "PORTAGE_REPOSITORIES" in settings:
 907                 repoconfigpaths.append(io.StringIO(settings["PORTAGE_REPOSITORIES"]))
 908         else:
 909                 if portage._working_copy:
 910                         repoconfigpaths.append(os.path.join(PORTAGE_BASE_PATH, "cnf", "repos.conf"))
 911                 else:
 912                         repoconfigpaths.append(os.path.join(settings.global_config_path, "repos.conf"))
 913                 repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "repos.conf"))
 914         if extra_files:
 915                 repoconfigpaths.extend(extra_files)
 916         return RepoConfigLoader(repoconfigpaths, settings)
 917
 918 def _get_repo_name(repo_location, cached=None):
 919         if cached is not None:
 920                 return cached
 921         name, missing = RepoConfig._read_repo_name(repo_location)
 922         if missing:
 923                 return None
 924         return name
 925
 926 def parse_layout_conf(repo_location, repo_name=None):
 927         eapi = read_corresponding_eapi_file(os.path.join(repo_location, REPO_NAME_LOC))
 928
 929         layout_filename = os.path.join(repo_location, "metadata", "layout.conf")
 930         layout_file = KeyValuePairFileLoader(layout_filename, None, None)
 931         layout_data, layout_errors = layout_file.load()
 932
 933         data = {}
 934
 935         # None indicates abscence of a masters setting, which later code uses
 936         # to trigger a backward compatibility fallback that sets an implicit
 937         # master. In order to avoid this fallback behavior, layout.conf can
 938         # explicitly set masters to an empty value, which will result in an
 939         # empty tuple here instead of None.
 940         masters = layout_data.get('masters')
 941         if masters is not None:
 942                 masters = tuple(masters.split())
 943         data['masters'] = masters
 944         data['aliases'] = tuple(layout_data.get('aliases', '').split())
 945
 946         data['allow-provide-virtual'] = \
 947                 layout_data.get('allow-provide-virtuals', 'false').lower() == 'true'
 948
 949         data['eapis-banned'] = tuple(layout_data.get('eapis-banned', '').split())
 950         data['eapis-deprecated'] = tuple(layout_data.get('eapis-deprecated', '').split())
 951
 952         data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \
 953                 == 'true'
 954
 955         data['sign-manifest'] = layout_data.get('sign-manifests', 'true').lower() \
 956                 == 'true'
 957
 958         data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \
 959                 == 'true'
 960
 961         data['repo-name'] = _gen_valid_repo(layout_data.get('repo-name', ''))
 962
 963         manifest_policy = layout_data.get('use-manifests', 'strict').lower()
 964         data['allow-missing-manifest'] = manifest_policy != 'strict'
 965         data['create-manifest'] = manifest_policy != 'false'
 966         data['disable-manifest'] = manifest_policy == 'false'
 967
 968         # for compatibility w/ PMS, fallback to pms; but also check if the
 969         # cache exists or not.
 970         cache_formats = layout_data.get('cache-formats', '').lower().split()
 971         if not cache_formats:
 972                 # Auto-detect cache formats, and prefer md5-cache if available.
 973                 # This behavior was deployed in portage-2.1.11.14, so that the
 974                 # default egencache format could eventually be changed to md5-dict
 975                 # in portage-2.1.11.32. WARNING: Versions prior to portage-2.1.11.14
 976                 # will NOT recognize md5-dict format unless it is explicitly
 977                 # listed in layout.conf.
 978                 cache_formats = []
 979                 if os.path.isdir(os.path.join(repo_location, 'metadata', 'md5-cache')):
 980                         cache_formats.append('md5-dict')
 981                 if os.path.isdir(os.path.join(repo_location, 'metadata', 'cache')):
 982                         cache_formats.append('pms')
 983         data['cache-formats'] = tuple(cache_formats)
 984
 985         manifest_hashes = layout_data.get('manifest-hashes')
 986         if manifest_hashes is not None:
 987                 manifest_hashes = frozenset(manifest_hashes.upper().split())
 988                 if MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 989                         repo_name = _get_repo_name(repo_location, cached=repo_name)
 990                         warnings.warn((_("Repository named '%(repo_name)s' has a "
 991                                 "'manifest-hashes' setting that does not contain "
 992                                 "the '%(hash)s' hash which is required by this "
 993                                 "portage version. You will have to upgrade portage "
 994                                 "if you want to generate valid manifests for this "
 995                                 "repository: %(layout_filename)s") %
 996                                 {"repo_name": repo_name or 'unspecified',
 997                                 "hash":MANIFEST2_REQUIRED_HASH,
 998                                 "layout_filename":layout_filename}),
 999                                 DeprecationWarning)
1000                 unsupported_hashes = manifest_hashes.difference(
1001                         MANIFEST2_HASH_FUNCTIONS)
1002                 if unsupported_hashes:
1003                         repo_name = _get_repo_name(repo_location, cached=repo_name)
1004                         warnings.warn((_("Repository named '%(repo_name)s' has a "
1005                                 "'manifest-hashes' setting that contains one "
1006                                 "or more hash types '%(hashes)s' which are not supported by "
1007                                 "this portage version. You will have to upgrade "
1008                                 "portage if you want to generate valid manifests for "
1009                                 "this repository: %(layout_filename)s") %
1010                                 {"repo_name": repo_name or 'unspecified',
1011                                 "hashes":" ".join(sorted(unsupported_hashes)),
1012                                 "layout_filename":layout_filename}),
1013                                 DeprecationWarning)
1014         data['manifest-hashes'] = manifest_hashes
1015
1016         data['update-changelog'] = layout_data.get('update-changelog', 'false').lower() \
1017                 == 'true'
1018
1019         raw_formats = layout_data.get('profile-formats')
1020         if raw_formats is None:
1021                 if eapi_allows_directories_on_profile_level_and_repository_level(eapi):
1022                         raw_formats = ('portage-1',)
1023                 else:
1024                         raw_formats = ('portage-1-compat',)
1025         else:
1026                 raw_formats = set(raw_formats.split())
1027                 unknown = raw_formats.difference(_valid_profile_formats)
1028                 if unknown:
1029                         repo_name = _get_repo_name(repo_location, cached=repo_name)
1030                         warnings.warn((_("Repository named '%(repo_name)s' has unsupported "
1031                                 "profiles in use ('profile-formats = %(unknown_fmts)s' setting in "
1032                                 "'%(layout_filename)s; please upgrade portage.") %
1033                                 dict(repo_name=repo_name or 'unspecified',
1034                                 layout_filename=layout_filename,
1035                                 unknown_fmts=" ".join(unknown))),
1036                                 DeprecationWarning)
1037                 raw_formats = tuple(raw_formats.intersection(_valid_profile_formats))
1038         data['profile-formats'] = raw_formats
1039
1040         return data, layout_errors