pym/portage/repository/config.py

   1 # Copyright 2010-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 import io
   7 import logging
   8 import warnings
   9 import sys
  10 import re
  11
  12 try:
  13         from configparser import Error as ConfigParserError
  14         if sys.hexversion >= 0x3020000:
  15                 from configparser import ConfigParser as SafeConfigParser
  16         else:
  17                 from configparser import SafeConfigParser
  18 except ImportError:
  19         from ConfigParser import SafeConfigParser, Error as ConfigParserError
  20 import portage
  21 from portage import eclass_cache, os
  22 from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH,
  23         REPO_NAME_LOC, USER_CONFIG_PATH)
  24 from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level
  25 from portage.env.loaders import KeyValuePairFileLoader
  26 from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split,
  27         stack_lists, writemsg, writemsg_level)
  28 from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess
  29 from portage.localization import _
  30 from portage import _unicode_decode
  31 from portage import _unicode_encode
  32 from portage import _encodings
  33 from portage import manifest
  34
  35 # Characters prohibited by repoman's file.name check.
  36 _invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]')
  37
  38 _valid_profile_formats = frozenset(
  39         ['pms', 'portage-1', 'portage-2'])
  40
  41 _portage1_profiles_allow_directories = frozenset(
  42         ["portage-1-compat", "portage-1", 'portage-2'])
  43
  44 _repo_name_sub_re = re.compile(r'[^\w-]')
  45
  46 def _gen_valid_repo(name):
  47         """
  48         Substitute hyphen in place of characters that don't conform to PMS 3.1.5,
  49         and strip hyphen from left side if necessary. This returns None if the
  50         given name contains no valid characters.
  51         """
  52         name = _repo_name_sub_re.sub(' ', name.strip())
  53         name = '-'.join(name.split())
  54         name = name.lstrip('-')
  55         if not name:
  56                 name = None
  57         return name
  58
  59 def _find_invalid_path_char(path, pos=0, endpos=None):
  60         """
  61         Returns the position of the first invalid character found in basename,
  62         or -1 if no invalid characters are found.
  63         """
  64         if endpos is None:
  65                 endpos = len(path)
  66
  67         m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos)
  68         if m is not None:
  69                 return m.start()
  70
  71         return -1
  72
  73 class RepoConfig(object):
  74         """Stores config of one repository"""
  75
  76         __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual',
  77                 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi',
  78                 'eclass_db', 'eclass_locations', 'eclass_overrides',
  79                 'find_invalid_path_char', 'format', 'local_config', 'location',
  80                 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
  81                 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
  82                 'profile_formats', 'sign_commit', 'sign_manifest', 'sync',
  83                 'thin_manifest', 'update_changelog', 'user_location',
  84                 '_eapis_banned', '_eapis_deprecated')
  85
  86         def __init__(self, name, repo_opts, local_config=True):
  87                 """Build a RepoConfig with options in repo_opts
  88                    Try to read repo_name in repository location, but if
  89                    it is not found use variable name as repository name"""
  90
  91                 self.local_config = local_config
  92
  93                 if local_config:
  94                         aliases = repo_opts.get('aliases')
  95                         if aliases is not None:
  96                                 aliases = tuple(aliases.split())
  97                 else:
  98                         aliases = None
  99
 100                 self.aliases = aliases
 101
 102                 if local_config:
 103                         eclass_overrides = repo_opts.get('eclass-overrides')
 104                         if eclass_overrides is not None:
 105                                 eclass_overrides = tuple(eclass_overrides.split())
 106                 else:
 107                         eclass_overrides = None
 108
 109                 self.eclass_overrides = eclass_overrides
 110                 # Eclass databases and locations are computed later.
 111                 self.eclass_db = None
 112                 self.eclass_locations = None
 113
 114                 if local_config:
 115                         # Masters from repos.conf override layout.conf.
 116                         masters = repo_opts.get('masters')
 117                         if masters is not None:
 118                                 masters = tuple(masters.split())
 119                 else:
 120                         masters = None
 121
 122                 self.masters = masters
 123
 124                 #The main-repo key makes only sense for the 'DEFAULT' section.
 125                 self.main_repo = repo_opts.get('main-repo')
 126
 127                 priority = repo_opts.get('priority')
 128                 if priority is not None:
 129                         try:
 130                                 priority = int(priority)
 131                         except ValueError:
 132                                 priority = None
 133                 self.priority = priority
 134
 135                 sync = repo_opts.get('sync')
 136                 if sync is not None:
 137                         sync = sync.strip()
 138                 self.sync = sync
 139
 140                 format = repo_opts.get('format')
 141                 if format is not None:
 142                         format = format.strip()
 143                 self.format = format
 144
 145                 location = repo_opts.get('location')
 146                 self.user_location = location
 147                 if location is not None and location.strip():
 148                         if os.path.isdir(location):
 149                                 location = os.path.realpath(location)
 150                 else:
 151                         location = None
 152                 self.location = location
 153
 154                 eapi = None
 155                 missing = True
 156                 self.name = name
 157                 if self.location is not None:
 158                         eapi = read_corresponding_eapi_file(os.path.join(self.location, REPO_NAME_LOC))
 159                         self.name, missing = self._read_valid_repo_name(self.location)
 160                 elif name == "DEFAULT":
 161                         missing = False
 162
 163                 self.eapi = eapi
 164                 self.missing_repo_name = missing
 165                 # sign_commit is disabled by default, since it requires Git >=1.7.9,
 166                 # and key_id configured by `git config user.signingkey key_id`
 167                 self.sign_commit = False
 168                 self.sign_manifest = True
 169                 self.thin_manifest = False
 170                 self.allow_missing_manifest = False
 171                 self.allow_provide_virtual = False
 172                 self.create_manifest = True
 173                 self.disable_manifest = False
 174                 self.manifest_hashes = None
 175                 self.update_changelog = False
 176                 self.cache_formats = None
 177                 self.portage1_profiles = True
 178                 self.portage1_profiles_compat = False
 179                 self.find_invalid_path_char = _find_invalid_path_char
 180
 181                 # Parse layout.conf.
 182                 if self.location:
 183                         layout_filename = os.path.join(self.location, "metadata", "layout.conf")
 184                         layout_data = parse_layout_conf(self.location, self.name)[0]
 185
 186                         # layout.conf masters may be overridden here if we have a masters
 187                         # setting from the user's repos.conf
 188                         if self.masters is None:
 189                                 self.masters = layout_data['masters']
 190
 191                         if local_config and layout_data['aliases']:
 192                                 aliases = self.aliases
 193                                 if aliases is None:
 194                                         aliases = ()
 195                                 # repos.conf aliases come after layout.conf aliases, giving
 196                                 # them the ability to do incremental overrides
 197                                 self.aliases = layout_data['aliases'] + tuple(aliases)
 198
 199                         if layout_data['repo-name']:
 200                                 # allow layout.conf to override repository name
 201                                 # useful when having two copies of the same repo enabled
 202                                 # to avoid modifying profiles/repo_name in one of them
 203                                 self.name = layout_data['repo-name']
 204
 205                         for value in ('allow-missing-manifest',
 206                                 'allow-provide-virtual', 'cache-formats',
 207                                 'create-manifest', 'disable-manifest', 'manifest-hashes',
 208                                 'profile-formats',
 209                                 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'):
 210                                 setattr(self, value.lower().replace("-", "_"), layout_data[value])
 211
 212                         self.portage1_profiles = eapi_allows_directories_on_profile_level_and_repository_level(eapi) or \
 213                                 any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats'])
 214                         self.portage1_profiles_compat = not eapi_allows_directories_on_profile_level_and_repository_level(eapi) and \
 215                                 layout_data['profile-formats'] == ('portage-1-compat',)
 216
 217                         self._eapis_banned = frozenset(layout_data['eapis-banned'])
 218                         self._eapis_deprecated = frozenset(layout_data['eapis-deprecated'])
 219
 220         def eapi_is_banned(self, eapi):
 221                 return eapi in self._eapis_banned
 222
 223         def eapi_is_deprecated(self, eapi):
 224                 return eapi in self._eapis_deprecated
 225
 226         def iter_pregenerated_caches(self, auxdbkeys, readonly=True, force=False):
 227                 """
 228                 Reads layout.conf cache-formats from left to right and yields cache
 229                 instances for each supported type that's found. If no cache-formats
 230                 are specified in layout.conf, 'pms' type is assumed if the
 231                 metadata/cache directory exists or force is True.
 232                 """
 233                 formats = self.cache_formats
 234                 if not formats:
 235                         if not force:
 236                                 return
 237                         # The default egencache format was 'pms' prior to portage-2.1.11.32
 238                         # (portage versions prior to portage-2.1.11.14 will NOT
 239                         # recognize md5-dict format unless it is explicitly listed in
 240                         # layout.conf).
 241                         formats = ('md5-dict',)
 242
 243                 for fmt in formats:
 244                         name = None
 245                         if fmt == 'pms':
 246                                 from portage.cache.metadata import database
 247                                 name = 'metadata/cache'
 248                         elif fmt == 'md5-dict':
 249                                 from portage.cache.flat_hash import md5_database as database
 250                                 name = 'metadata/md5-cache'
 251
 252                         if name is not None:
 253                                 yield database(self.location, name,
 254                                         auxdbkeys, readonly=readonly)
 255
 256         def get_pregenerated_cache(self, auxdbkeys, readonly=True, force=False):
 257                 """
 258                 Returns the first cache instance yielded from
 259                 iter_pregenerated_caches(), or None if no cache is available or none
 260                 of the available formats are supported.
 261                 """
 262                 return next(self.iter_pregenerated_caches(
 263                         auxdbkeys, readonly=readonly, force=force), None)
 264
 265         def load_manifest(self, *args, **kwds):
 266                 kwds['thin'] = self.thin_manifest
 267                 kwds['allow_missing'] = self.allow_missing_manifest
 268                 kwds['allow_create'] = self.create_manifest
 269                 kwds['hashes'] = self.manifest_hashes
 270                 if self.disable_manifest:
 271                         kwds['from_scratch'] = True
 272                 kwds['find_invalid_path_char'] = self.find_invalid_path_char
 273                 return manifest.Manifest(*args, **portage._native_kwargs(kwds))
 274
 275         def update(self, new_repo):
 276                 """Update repository with options in another RepoConfig"""
 277
 278                 keys = set(self.__slots__)
 279                 keys.discard("missing_repo_name")
 280                 for k in keys:
 281                         v = getattr(new_repo, k, None)
 282                         if v is not None:
 283                                 setattr(self, k, v)
 284
 285                 if new_repo.name is not None:
 286                         self.missing_repo_name = new_repo.missing_repo_name
 287
 288         @staticmethod
 289         def _read_valid_repo_name(repo_path):
 290                 name, missing = RepoConfig._read_repo_name(repo_path)
 291                 # We must ensure that the name conforms to PMS 3.1.5
 292                 # in order to avoid InvalidAtom exceptions when we
 293                 # use it to generate atoms.
 294                 name = _gen_valid_repo(name)
 295                 if not name:
 296                         # name only contains invalid characters
 297                         name = "x-" + os.path.basename(repo_path)
 298                         name = _gen_valid_repo(name)
 299                         # If basename only contains whitespace then the
 300                         # end result is name = 'x-'.
 301                 return name, missing
 302
 303         @staticmethod
 304         def _read_repo_name(repo_path):
 305                 """
 306                 Read repo_name from repo_path.
 307                 Returns repo_name, missing.
 308                 """
 309                 repo_name_path = os.path.join(repo_path, REPO_NAME_LOC)
 310                 f = None
 311                 try:
 312                         f = io.open(
 313                                 _unicode_encode(repo_name_path,
 314                                 encoding=_encodings['fs'], errors='strict'),
 315                                 mode='r', encoding=_encodings['repo.content'],
 316                                 errors='replace')
 317                         return f.readline().strip(), False
 318                 except EnvironmentError:
 319                         return "x-" + os.path.basename(repo_path), True
 320                 finally:
 321                         if f is not None:
 322                                 f.close()
 323
 324         def info_string(self):
 325                 """
 326                 Returns a formatted string containing informations about the repository.
 327                 Used by emerge --info.
 328                 """
 329                 indent = " " * 4
 330                 repo_msg = []
 331                 repo_msg.append(self.name)
 332                 if self.format:
 333                         repo_msg.append(indent + "format: " + self.format)
 334                 if self.user_location:
 335                         repo_msg.append(indent + "location: " + self.user_location)
 336                 if self.sync:
 337                         repo_msg.append(indent + "sync: " + self.sync)
 338                 if self.masters:
 339                         repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters))
 340                 if self.priority is not None:
 341                         repo_msg.append(indent + "priority: " + str(self.priority))
 342                 if self.aliases:
 343                         repo_msg.append(indent + "aliases: " + " ".join(self.aliases))
 344                 if self.eclass_overrides:
 345                         repo_msg.append(indent + "eclass_overrides: " + \
 346                                 " ".join(self.eclass_overrides))
 347                 repo_msg.append("")
 348                 return "\n".join(repo_msg)
 349
 350         def __repr__(self):
 351                 return "<portage.repository.config.RepoConfig(name='%s', location='%s')>" % (self.name, _unicode_decode(self.location))
 352
 353         def __str__(self):
 354                 d = {}
 355                 for k in self.__slots__:
 356                         d[k] = getattr(self, k, None)
 357                 return "%s" % (d,)
 358
 359         if sys.hexversion < 0x3000000:
 360
 361                 __unicode__ = __str__
 362
 363                 def __str__(self):
 364                         return _unicode_encode(self.__unicode__())
 365
 366 class RepoConfigLoader(object):
 367         """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf"""
 368
 369         @staticmethod
 370         def _add_repositories(portdir, portdir_overlay, prepos,
 371                 ignored_map, ignored_location_map, local_config):
 372                 """Add overlays in PORTDIR_OVERLAY as repositories"""
 373                 overlays = []
 374                 portdir_orig = None
 375                 if portdir:
 376                         portdir = normalize_path(portdir)
 377                         portdir_orig = portdir
 378                         overlays.append(portdir)
 379                 try:
 380                         port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
 381                 except ValueError as e:
 382                         #File "/usr/lib/python3.2/shlex.py", line 168, in read_token
 383                         #       raise ValueError("No closing quotation")
 384                         writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
 385                                 " %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
 386                         port_ov = []
 387                 overlays.extend(port_ov)
 388                 default_repo_opts = {}
 389                 if prepos['DEFAULT'].aliases is not None:
 390                         default_repo_opts['aliases'] = \
 391                                 ' '.join(prepos['DEFAULT'].aliases)
 392                 if prepos['DEFAULT'].eclass_overrides is not None:
 393                         default_repo_opts['eclass-overrides'] = \
 394                                 ' '.join(prepos['DEFAULT'].eclass_overrides)
 395                 if prepos['DEFAULT'].masters is not None:
 396                         default_repo_opts['masters'] = \
 397                                 ' '.join(prepos['DEFAULT'].masters)
 398
 399                 if overlays:
 400                         # We need a copy of the original repos.conf data, since we're
 401                         # going to modify the prepos dict and some of the RepoConfig
 402                         # objects that we put in prepos may have to be discarded if
 403                         # they get overridden by a repository with the same name but
 404                         # a different location. This is common with repoman, for example,
 405                         # when temporarily overriding an rsync repo with another copy
 406                         # of the same repo from CVS.
 407                         repos_conf = prepos.copy()
 408                         #overlay priority is negative because we want them to be looked before any other repo
 409                         base_priority = 0
 410                         for ov in overlays:
 411                                 if isdir_raise_eaccess(ov):
 412                                         repo_opts = default_repo_opts.copy()
 413                                         repo_opts['location'] = ov
 414                                         repo = RepoConfig(None, repo_opts, local_config=local_config)
 415                                         # repos_conf_opts contains options from repos.conf
 416                                         repos_conf_opts = repos_conf.get(repo.name)
 417                                         if repos_conf_opts is not None:
 418                                                 # Selectively copy only the attributes which
 419                                                 # repos.conf is allowed to override.
 420                                                 for k in ('aliases', 'eclass_overrides', 'masters', 'priority'):
 421                                                         v = getattr(repos_conf_opts, k, None)
 422                                                         if v is not None:
 423                                                                 setattr(repo, k, v)
 424
 425                                         if repo.name in prepos:
 426                                                 old_location = prepos[repo.name].location
 427                                                 if old_location is not None and old_location != repo.location:
 428                                                         ignored_map.setdefault(repo.name, []).append(old_location)
 429                                                         ignored_location_map[old_location] = repo.name
 430                                                         if old_location == portdir:
 431                                                                 portdir = repo.user_location
 432
 433                                         if repo.priority is None:
 434                                                 if base_priority == 0 and ov == portdir_orig:
 435                                                         # If it's the original PORTDIR setting and it's not
 436                                                         # in PORTDIR_OVERLAY, then it will be assigned a
 437                                                         # special priority setting later.
 438                                                         pass
 439                                                 else:
 440                                                         repo.priority = base_priority
 441                                                         base_priority += 1
 442
 443                                         prepos[repo.name] = repo
 444                                 else:
 445                                         if base_priority == 0 and ov == '/usr/portage':
 446                                                 # Suppress warnings for the make.globals
 447                                                 # PORTDIR setting if we have an existing
 448                                                 # main-repo defined in repos.conf.
 449                                                 main_repo = prepos['DEFAULT'].main_repo
 450                                                 if main_repo is not None and main_repo in prepos:
 451                                                         main_repo_loc = prepos[main_repo].location
 452                                                         if main_repo_loc and \
 453                                                                 isdir_raise_eaccess(main_repo_loc):
 454                                                                 continue
 455
 456                                         if not portage._sync_disabled_warnings:
 457                                                 writemsg(_("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n") % ov, noiselevel=-1)
 458
 459                 return portdir
 460
 461         @staticmethod
 462         def _parse(paths, prepos, ignored_map, ignored_location_map, local_config):
 463                 """Parse files in paths to load config"""
 464                 parser = SafeConfigParser()
 465
 466                 # use read_file/readfp in order to control decoding of unicode
 467                 try:
 468                         # Python >=3.2
 469                         read_file = parser.read_file
 470                         source_kwarg = 'source'
 471                 except AttributeError:
 472                         read_file = parser.readfp
 473                         source_kwarg = 'filename'
 474
 475                 for p in paths:
 476                         f = None
 477                         try:
 478                                 f = io.open(_unicode_encode(p,
 479                                         encoding=_encodings['fs'], errors='strict'),
 480                                         mode='r', encoding=_encodings['repo.content'],
 481                                         errors='replace')
 482                         except EnvironmentError:
 483                                 pass
 484                         else:
 485                                 # The 'source' keyword argument is needed since
 486                                 # otherwise ConfigParsier may throw a TypeError because
 487                                 # it assumes that f.name is a native string rather
 488                                 # than binary when constructing error messages.
 489                                 kwargs = {source_kwarg: p}
 490                                 read_file(f, **portage._native_kwargs(kwargs))
 491                         finally:
 492                                 if f is not None:
 493                                         f.close()
 494
 495                 prepos['DEFAULT'] = RepoConfig("DEFAULT",
 496                         parser.defaults(), local_config=local_config)
 497
 498                 for sname in parser.sections():
 499                         optdict = {}
 500                         for oname in parser.options(sname):
 501                                 optdict[oname] = parser.get(sname, oname)
 502
 503                         repo = RepoConfig(sname, optdict, local_config=local_config)
 504                         if repo.name != sname:
 505                                 writemsg_level("!!! %s\n" %
 506                                    _("Section name '%s' set in repos.conf differs from name '%s' set inside repository") %
 507                                         (sname, repo.name), level=logging.ERROR, noiselevel=-1)
 508                                 continue
 509
 510                         if repo.location and not exists_raise_eaccess(repo.location):
 511                                 writemsg(_("!!! Invalid repos.conf entry '%s'"
 512                                         " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1)
 513                                 continue
 514
 515                         if repo.name in prepos:
 516                                 old_location = prepos[repo.name].location
 517                                 if old_location is not None and repo.location is not None and old_location != repo.location:
 518                                         ignored_map.setdefault(repo.name, []).append(old_location)
 519                                         ignored_location_map[old_location] = repo.name
 520                                 prepos[repo.name].update(repo)
 521                         else:
 522                                 prepos[repo.name] = repo
 523
 524         def __init__(self, paths, settings):
 525                 """Load config from files in paths"""
 526
 527                 prepos = {}
 528                 location_map = {}
 529                 treemap = {}
 530                 ignored_map = {}
 531                 ignored_location_map = {}
 532
 533                 portdir = settings.get('PORTDIR', '')
 534                 portdir_overlay = settings.get('PORTDIR_OVERLAY', '')
 535
 536                 try:
 537                         self._parse(paths, prepos, ignored_map,
 538                                 ignored_location_map, settings.local_config)
 539                 except ConfigParserError as e:
 540                         writemsg(
 541                                 _("!!! Error while reading repo config file: %s\n") % e,
 542                                 noiselevel=-1)
 543                         # The configparser state is unreliable (prone to quirky
 544                         # exceptions) after it has thrown an error, so use empty
 545                         # config and try to fall back to PORTDIR{,_OVERLAY}.
 546                         prepos.clear()
 547                         prepos['DEFAULT'] = RepoConfig('DEFAULT',
 548                                 {}, local_config=settings.local_config)
 549                         location_map.clear()
 550                         treemap.clear()
 551                         ignored_map.clear()
 552                         ignored_location_map.clear()
 553
 554                 # If PORTDIR_OVERLAY contains a repo with the same repo_name as
 555                 # PORTDIR, then PORTDIR is overridden.
 556                 portdir = self._add_repositories(portdir, portdir_overlay, prepos,
 557                         ignored_map, ignored_location_map, settings.local_config)
 558                 if portdir and portdir.strip():
 559                         portdir = os.path.realpath(portdir)
 560
 561                 ignored_repos = tuple((repo_name, tuple(paths)) \
 562                         for repo_name, paths in ignored_map.items())
 563
 564                 self.missing_repo_names = frozenset(repo.location
 565                         for repo in prepos.values()
 566                         if repo.location is not None and repo.missing_repo_name)
 567
 568                 # Do this before expanding aliases, so that location_map and
 569                 # treemap consistently map unaliased names whenever available.
 570                 for repo_name, repo in list(prepos.items()):
 571                         if repo.location is None:
 572                                 if repo_name != 'DEFAULT':
 573                                         # Skip this warning for repoman (bug #474578).
 574                                         if settings.local_config and paths:
 575                                                 writemsg_level(_("Location undefined for " \
 576                                                         "repository '%s' referenced in '%s'\n") % \
 577                                                         (repo.name, paths[0]),
 578                                                         level=logging.ERROR, noiselevel=-1)
 579                                         del prepos[repo_name]
 580                         else:
 581                                 location_map[repo.location] = repo_name
 582                                 treemap[repo_name] = repo.location
 583
 584                 # Add alias mappings, but never replace unaliased mappings.
 585                 for repo_name, repo in list(prepos.items()):
 586                         names = set()
 587                         names.add(repo_name)
 588                         if repo.aliases:
 589                                 aliases = stack_lists([repo.aliases], incremental=True)
 590                                 names.update(aliases)
 591
 592                         for name in names:
 593                                 if name in prepos and prepos[name].location is not None:
 594                                         if name == repo_name:
 595                                                 # unaliased names already handled earlier
 596                                                 continue
 597                                         writemsg_level(_("!!! Repository name or alias '%s', " + \
 598                                                 "defined for repository '%s', overrides " + \
 599                                                 "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1)
 600                                         # Never replace an unaliased mapping with
 601                                         # an aliased mapping.
 602                                         continue
 603                                 prepos[name] = repo
 604                                 if repo.location is not None:
 605                                         if repo.location not in location_map:
 606                                                 # Never replace an unaliased mapping with
 607                                                 # an aliased mapping.
 608                                                 location_map[repo.location] = name
 609                                         treemap[name] = repo.location
 610
 611                 main_repo = prepos['DEFAULT'].main_repo
 612                 if main_repo is None or main_repo not in prepos:
 613                         #setting main_repo if it was not set in repos.conf
 614                         main_repo = location_map.get(portdir)
 615                         if main_repo is not None:
 616                                 prepos['DEFAULT'].main_repo = main_repo
 617                         else:
 618                                 prepos['DEFAULT'].main_repo = None
 619                                 if not portage._sync_disabled_warnings:
 620                                         writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty.\n"), noiselevel=-1)
 621
 622                 if main_repo is not None and prepos[main_repo].priority is None:
 623                         # This happens if main-repo has been set in repos.conf.
 624                         prepos[main_repo].priority = -1000
 625
 626                 # Include repo.name in sort key, for predictable sorting
 627                 # even when priorities are equal.
 628                 prepos_order = sorted(prepos.items(),
 629                         key=lambda r:(r[1].priority or 0, r[1].name))
 630
 631                 # filter duplicates from aliases, by only including
 632                 # items where repo.name == key
 633                 prepos_order = [repo.name for (key, repo) in prepos_order
 634                         if repo.name == key and key != 'DEFAULT' and
 635                         repo.location is not None]
 636
 637                 self.prepos = prepos
 638                 self.prepos_order = prepos_order
 639                 self.ignored_repos = ignored_repos
 640                 self.location_map = location_map
 641                 self.treemap = treemap
 642                 self._prepos_changed = True
 643                 self._repo_location_list = []
 644
 645                 #The 'masters' key currently contains repo names. Replace them with the matching RepoConfig.
 646                 for repo_name, repo in prepos.items():
 647                         if repo_name == "DEFAULT":
 648                                 continue
 649                         if repo.masters is None:
 650                                 if self.mainRepo() and repo_name != self.mainRepo().name:
 651                                         repo.masters = self.mainRepo(),
 652                                 else:
 653                                         repo.masters = ()
 654                         else:
 655                                 if repo.masters and isinstance(repo.masters[0], RepoConfig):
 656                                         # This one has already been processed
 657                                         # because it has an alias.
 658                                         continue
 659                                 master_repos = []
 660                                 for master_name in repo.masters:
 661                                         if master_name not in prepos:
 662                                                 layout_filename = os.path.join(repo.user_location,
 663                                                         "metadata", "layout.conf")
 664                                                 writemsg_level(_("Unavailable repository '%s' " \
 665                                                         "referenced by masters entry in '%s'\n") % \
 666                                                         (master_name, layout_filename),
 667                                                         level=logging.ERROR, noiselevel=-1)
 668                                         else:
 669                                                 master_repos.append(prepos[master_name])
 670                                 repo.masters = tuple(master_repos)
 671
 672                 #The 'eclass_overrides' key currently contains repo names. Replace them with the matching repo paths.
 673                 for repo_name, repo in prepos.items():
 674                         if repo_name == "DEFAULT":
 675                                 continue
 676
 677                         eclass_locations = []
 678                         eclass_locations.extend(master_repo.location for master_repo in repo.masters)
 679                         # Only append the current repo to eclass_locations if it's not
 680                         # there already. This allows masters to have more control over
 681                         # eclass override order, which may be useful for scenarios in
 682                         # which there is a plan to migrate eclasses to a master repo.
 683                         if repo.location not in eclass_locations:
 684                                 eclass_locations.append(repo.location)
 685
 686                         if repo.eclass_overrides:
 687                                 for other_repo_name in repo.eclass_overrides:
 688                                         if other_repo_name in self.treemap:
 689                                                 eclass_locations.append(self.get_location_for_name(other_repo_name))
 690                                         else:
 691                                                 writemsg_level(_("Unavailable repository '%s' " \
 692                                                         "referenced by eclass-overrides entry for " \
 693                                                         "'%s'\n") % (other_repo_name, repo_name), \
 694                                                         level=logging.ERROR, noiselevel=-1)
 695                         repo.eclass_locations = tuple(eclass_locations)
 696
 697                 eclass_dbs = {}
 698                 for repo_name, repo in prepos.items():
 699                         if repo_name == "DEFAULT":
 700                                 continue
 701
 702                         eclass_db = None
 703                         for eclass_location in repo.eclass_locations:
 704                                 tree_db = eclass_dbs.get(eclass_location)
 705                                 if tree_db is None:
 706                                         tree_db = eclass_cache.cache(eclass_location)
 707                                         eclass_dbs[eclass_location] = tree_db
 708                                 if eclass_db is None:
 709                                         eclass_db = tree_db.copy()
 710                                 else:
 711                                         eclass_db.append(tree_db)
 712                         repo.eclass_db = eclass_db
 713
 714                 self._prepos_changed = True
 715                 self._repo_location_list = []
 716
 717                 self._check_locations()
 718
 719         def repoLocationList(self):
 720                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
 721                 if self._prepos_changed:
 722                         _repo_location_list = []
 723                         for repo in self.prepos_order:
 724                                 if self.prepos[repo].location is not None:
 725                                         _repo_location_list.append(self.prepos[repo].location)
 726                         self._repo_location_list = tuple(_repo_location_list)
 727
 728                         self._prepos_changed = False
 729                 return self._repo_location_list
 730
 731         def repoUserLocationList(self):
 732                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
 733                 user_location_list = []
 734                 for repo in self.prepos_order:
 735                         if self.prepos[repo].location is not None:
 736                                 user_location_list.append(self.prepos[repo].user_location)
 737                 return tuple(user_location_list)
 738
 739         def mainRepoLocation(self):
 740                 """Returns the location of main repo"""
 741                 main_repo = self.prepos['DEFAULT'].main_repo
 742                 if main_repo is not None and main_repo in self.prepos:
 743                         return self.prepos[main_repo].location
 744                 else:
 745                         return ''
 746
 747         def mainRepo(self):
 748                 """Returns the main repo"""
 749                 main_repo = self.prepos['DEFAULT'].main_repo
 750                 if main_repo is None:
 751                         return None
 752                 return self.prepos[main_repo]
 753
 754         def _check_locations(self):
 755                 """Check if repositories location are correct and show a warning message if not"""
 756                 for (name, r) in self.prepos.items():
 757                         if name != 'DEFAULT':
 758                                 if r.location is None:
 759                                         writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1)
 760                                 else:
 761                                         if not isdir_raise_eaccess(r.location):
 762                                                 self.prepos_order.remove(name)
 763                                                 writemsg(_("!!! Invalid Repository Location"
 764                                                         " (not a dir): '%s'\n") % r.location, noiselevel=-1)
 765
 766         def repos_with_profiles(self):
 767                 for repo_name in self.prepos_order:
 768                         repo = self.prepos[repo_name]
 769                         if repo.format != "unavailable":
 770                                 yield repo
 771
 772         def get_name_for_location(self, location):
 773                 return self.location_map[location]
 774
 775         def get_location_for_name(self, repo_name):
 776                 if repo_name is None:
 777                         # This simplifies code in places where
 778                         # we want to be able to pass in Atom.repo
 779                         # even if it is None.
 780                         return None
 781                 return self.treemap[repo_name]
 782
 783         def get_repo_for_location(self, location):
 784                 return self.prepos[self.get_name_for_location(location)]
 785
 786         def __getitem__(self, repo_name):
 787                 return self.prepos[repo_name]
 788
 789         def __iter__(self):
 790                 for repo_name in self.prepos_order:
 791                         yield self.prepos[repo_name]
 792
 793 def load_repository_config(settings):
 794         #~ repoconfigpaths = [os.path.join(settings.global_config_path, "repos.conf")]
 795         repoconfigpaths = []
 796         repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
 797                 USER_CONFIG_PATH, "repos.conf"))
 798         return RepoConfigLoader(repoconfigpaths, settings)
 799
 800 def _get_repo_name(repo_location, cached=None):
 801         if cached is not None:
 802                 return cached
 803         name, missing = RepoConfig._read_repo_name(repo_location)
 804         if missing:
 805                 return None
 806         return name
 807
 808 def parse_layout_conf(repo_location, repo_name=None):
 809         eapi = read_corresponding_eapi_file(os.path.join(repo_location, REPO_NAME_LOC))
 810
 811         layout_filename = os.path.join(repo_location, "metadata", "layout.conf")
 812         layout_file = KeyValuePairFileLoader(layout_filename, None, None)
 813         layout_data, layout_errors = layout_file.load()
 814
 815         data = {}
 816
 817         # None indicates abscence of a masters setting, which later code uses
 818         # to trigger a backward compatibility fallback that sets an implicit
 819         # master. In order to avoid this fallback behavior, layout.conf can
 820         # explicitly set masters to an empty value, which will result in an
 821         # empty tuple here instead of None.
 822         masters = layout_data.get('masters')
 823         if masters is not None:
 824                 masters = tuple(masters.split())
 825         data['masters'] = masters
 826         data['aliases'] = tuple(layout_data.get('aliases', '').split())
 827
 828         data['allow-provide-virtual'] = \
 829                 layout_data.get('allow-provide-virtuals', 'false').lower() == 'true'
 830
 831         data['eapis-banned'] = tuple(layout_data.get('eapis-banned', '').split())
 832         data['eapis-deprecated'] = tuple(layout_data.get('eapis-deprecated', '').split())
 833
 834         data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \
 835                 == 'true'
 836
 837         data['sign-manifest'] = layout_data.get('sign-manifests', 'true').lower() \
 838                 == 'true'
 839
 840         data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \
 841                 == 'true'
 842
 843         data['repo-name'] = _gen_valid_repo(layout_data.get('repo-name', ''))
 844
 845         manifest_policy = layout_data.get('use-manifests', 'strict').lower()
 846         data['allow-missing-manifest'] = manifest_policy != 'strict'
 847         data['create-manifest'] = manifest_policy != 'false'
 848         data['disable-manifest'] = manifest_policy == 'false'
 849
 850         # for compatibility w/ PMS, fallback to pms; but also check if the
 851         # cache exists or not.
 852         cache_formats = layout_data.get('cache-formats', '').lower().split()
 853         if not cache_formats:
 854                 # Auto-detect cache formats, and prefer md5-cache if available.
 855                 # This behavior was deployed in portage-2.1.11.14, so that the
 856                 # default egencache format could eventually be changed to md5-dict
 857                 # in portage-2.1.11.32. WARNING: Versions prior to portage-2.1.11.14
 858                 # will NOT recognize md5-dict format unless it is explicitly
 859                 # listed in layout.conf.
 860                 cache_formats = []
 861                 if os.path.isdir(os.path.join(repo_location, 'metadata', 'md5-cache')):
 862                         cache_formats.append('md5-dict')
 863                 if os.path.isdir(os.path.join(repo_location, 'metadata', 'cache')):
 864                         cache_formats.append('pms')
 865         data['cache-formats'] = tuple(cache_formats)
 866
 867         manifest_hashes = layout_data.get('manifest-hashes')
 868         if manifest_hashes is not None:
 869                 manifest_hashes = frozenset(manifest_hashes.upper().split())
 870                 if MANIFEST2_REQUIRED_HASH not in manifest_hashes:
 871                         repo_name = _get_repo_name(repo_location, cached=repo_name)
 872                         warnings.warn((_("Repository named '%(repo_name)s' has a "
 873                                 "'manifest-hashes' setting that does not contain "
 874                                 "the '%(hash)s' hash which is required by this "
 875                                 "portage version. You will have to upgrade portage "
 876                                 "if you want to generate valid manifests for this "
 877                                 "repository: %(layout_filename)s") %
 878                                 {"repo_name": repo_name or 'unspecified',
 879                                 "hash":MANIFEST2_REQUIRED_HASH,
 880                                 "layout_filename":layout_filename}),
 881                                 DeprecationWarning)
 882                 unsupported_hashes = manifest_hashes.difference(
 883                         MANIFEST2_HASH_FUNCTIONS)
 884                 if unsupported_hashes:
 885                         repo_name = _get_repo_name(repo_location, cached=repo_name)
 886                         warnings.warn((_("Repository named '%(repo_name)s' has a "
 887                                 "'manifest-hashes' setting that contains one "
 888                                 "or more hash types '%(hashes)s' which are not supported by "
 889                                 "this portage version. You will have to upgrade "
 890                                 "portage if you want to generate valid manifests for "
 891                                 "this repository: %(layout_filename)s") %
 892                                 {"repo_name": repo_name or 'unspecified',
 893                                 "hashes":" ".join(sorted(unsupported_hashes)),
 894                                 "layout_filename":layout_filename}),
 895                                 DeprecationWarning)
 896         data['manifest-hashes'] = manifest_hashes
 897
 898         data['update-changelog'] = layout_data.get('update-changelog', 'false').lower() \
 899                 == 'true'
 900
 901         raw_formats = layout_data.get('profile-formats')
 902         if raw_formats is None:
 903                 if eapi_allows_directories_on_profile_level_and_repository_level(eapi):
 904                         raw_formats = ('portage-1',)
 905                 else:
 906                         raw_formats = ('portage-1-compat',)
 907         else:
 908                 raw_formats = set(raw_formats.split())
 909                 unknown = raw_formats.difference(_valid_profile_formats)
 910                 if unknown:
 911                         repo_name = _get_repo_name(repo_location, cached=repo_name)
 912                         warnings.warn((_("Repository named '%(repo_name)s' has unsupported "
 913                                 "profiles in use ('profile-formats = %(unknown_fmts)s' setting in "
 914                                 "'%(layout_filename)s; please upgrade portage.") %
 915                                 dict(repo_name=repo_name or 'unspecified',
 916                                 layout_filename=layout_filename,
 917                                 unknown_fmts=" ".join(unknown))),
 918                                 DeprecationWarning)
 919                 raw_formats = tuple(raw_formats.intersection(_valid_profile_formats))
 920         data['profile-formats'] = raw_formats
 921
 922         return data, layout_errors