pym/portage/dbapi/porttree.py

   1 # Copyright 1998-2012 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 __all__ = [
   5         "close_portdbapi_caches", "FetchlistDict", "portagetree", "portdbapi"
   6 ]
   7
   8 import portage
   9 portage.proxy.lazyimport.lazyimport(globals(),
  10         'portage.checksum',
  11         'portage.data:portage_gid,secpass',
  12         'portage.dbapi.dep_expand:dep_expand',
  13         'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot',
  14         'portage.package.ebuild.doebuild:doebuild',
  15         'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
  16         'portage.util.listdir:listdir',
  17         'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str',
  18 )
  19
  20 from portage.cache import volatile
  21 from portage.cache.cache_errors import CacheError
  22 from portage.cache.mappings import Mapping
  23 from portage.dbapi import dbapi
  24 from portage.exception import PortageException, \
  25         FileNotFound, InvalidAtom, InvalidData, \
  26         InvalidDependString, InvalidPackageName
  27 from portage.localization import _
  28
  29 from portage import eclass_cache, \
  30         eapi_is_supported, \
  31         _eapi_is_deprecated
  32 from portage import os
  33 from portage import _encodings
  34 from portage import _unicode_encode
  35 from portage import OrderedDict
  36 from portage.util._eventloop.EventLoop import EventLoop
  37 from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
  38
  39 import os as _os
  40 import sys
  41 import traceback
  42 import warnings
  43
  44 if sys.hexversion >= 0x3000000:
  45         basestring = str
  46         long = int
  47
  48 class portdbapi(dbapi):
  49         """this tree will scan a portage directory located at root (passed to init)"""
  50         portdbapi_instances = []
  51         _use_mutable = True
  52
  53         @property
  54         def _categories(self):
  55                 return self.settings.categories
  56
  57         @property
  58         def porttree_root(self):
  59                 return self.settings.repositories.mainRepoLocation()
  60
  61         @property
  62         def eclassdb(self):
  63                 main_repo = self.repositories.mainRepo()
  64                 if main_repo is None:
  65                         return None
  66                 return main_repo.eclass_db
  67
  68         def __init__(self, _unused_param=None, mysettings=None):
  69                 """
  70                 @param _unused_param: deprecated, use mysettings['PORTDIR'] instead
  71                 @type _unused_param: None
  72                 @param mysettings: an immutable config instance
  73                 @type mysettings: portage.config
  74                 """
  75                 portdbapi.portdbapi_instances.append(self)
  76
  77                 from portage import config
  78                 if mysettings:
  79                         self.settings = mysettings
  80                 else:
  81                         from portage import settings
  82                         self.settings = config(clone=settings)
  83
  84                 if _unused_param is not None:
  85                         warnings.warn("The first parameter of the " + \
  86                                 "portage.dbapi.porttree.portdbapi" + \
  87                                 " constructor is unused since portage-2.1.8. " + \
  88                                 "mysettings['PORTDIR'] is used instead.",
  89                                 DeprecationWarning, stacklevel=2)
  90
  91                 self.repositories = self.settings.repositories
  92                 self.treemap = self.repositories.treemap
  93
  94                 # This is strictly for use in aux_get() doebuild calls when metadata
  95                 # is generated by the depend phase.  It's safest to use a clone for
  96                 # this purpose because doebuild makes many changes to the config
  97                 # instance that is passed in.
  98                 self.doebuild_settings = config(clone=self.settings)
  99                 self._event_loop = EventLoop(main=False)
 100                 self.depcachedir = os.path.realpath(self.settings.depcachedir)
 101
 102                 if os.environ.get("SANDBOX_ON") == "1":
 103                         # Make api consumers exempt from sandbox violations
 104                         # when doing metadata cache updates.
 105                         sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":")
 106                         if self.depcachedir not in sandbox_write:
 107                                 sandbox_write.append(self.depcachedir)
 108                                 os.environ["SANDBOX_WRITE"] = \
 109                                         ":".join(filter(None, sandbox_write))
 110
 111                 self.porttrees = list(self.settings.repositories.repoLocationList())
 112
 113                 # This is used as sanity check for aux_get(). If there is no
 114                 # root eclass dir, we assume that PORTDIR is invalid or
 115                 # missing. This check allows aux_get() to detect a missing
 116                 # portage tree and return early by raising a KeyError.
 117                 self._have_root_eclass_dir = os.path.isdir(
 118                         os.path.join(self.settings.repositories.mainRepoLocation(), "eclass"))
 119
 120                 #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
 121                 self.xcache = {}
 122                 self.frozen = 0
 123
 124                 #Keep a list of repo names, sorted by priority (highest priority first).
 125                 self._ordered_repo_name_list = tuple(reversed(self.repositories.prepos_order))
 126
 127                 self.auxdbmodule = self.settings.load_best_module("portdbapi.auxdbmodule")
 128                 self.auxdb = {}
 129                 self._pregen_auxdb = {}
 130                 # If the current user doesn't have depcachedir write permission,
 131                 # then the depcachedir cache is kept here read-only access.
 132                 self._ro_auxdb = {}
 133                 self._init_cache_dirs()
 134                 try:
 135                         depcachedir_st = os.stat(self.depcachedir)
 136                         depcachedir_w_ok = os.access(self.depcachedir, os.W_OK)
 137                 except OSError:
 138                         depcachedir_st = None
 139                         depcachedir_w_ok = False
 140
 141                 cache_kwargs = {}
 142
 143                 depcachedir_unshared = False
 144                 if portage.data.secpass < 1 and \
 145                         depcachedir_w_ok and \
 146                         depcachedir_st is not None and \
 147                         os.getuid() == depcachedir_st.st_uid and \
 148                         os.getgid() == depcachedir_st.st_gid:
 149                         # If this user owns depcachedir and is not in the
 150                         # portage group, then don't bother to set permissions
 151                         # on cache entries. This makes it possible to run
 152                         # egencache without any need to be a member of the
 153                         # portage group.
 154                         depcachedir_unshared = True
 155                 else:
 156                         cache_kwargs.update({
 157                                 'gid'     : portage_gid,
 158                                 'perms'   : 0o664
 159                         })
 160
 161                 # If secpass < 1, we don't want to write to the cache
 162                 # since then we won't be able to apply group permissions
 163                 # to the cache entries/directories.
 164                 if (secpass < 1 and not depcachedir_unshared) or not depcachedir_w_ok:
 165                         for x in self.porttrees:
 166                                 self.auxdb[x] = volatile.database(
 167                                         self.depcachedir, x, self._known_keys,
 168                                         **cache_kwargs)
 169                                 try:
 170                                         self._ro_auxdb[x] = self.auxdbmodule(self.depcachedir, x,
 171                                                 self._known_keys, readonly=True, **cache_kwargs)
 172                                 except CacheError:
 173                                         pass
 174                 else:
 175                         for x in self.porttrees:
 176                                 if x in self.auxdb:
 177                                         continue
 178                                 # location, label, auxdbkeys
 179                                 self.auxdb[x] = self.auxdbmodule(
 180                                         self.depcachedir, x, self._known_keys, **cache_kwargs)
 181                 if "metadata-transfer" not in self.settings.features:
 182                         for x in self.porttrees:
 183                                 if x in self._pregen_auxdb:
 184                                         continue
 185                                 cache = self._create_pregen_cache(x)
 186                                 if cache is not None:
 187                                         self._pregen_auxdb[x] = cache
 188                 # Selectively cache metadata in order to optimize dep matching.
 189                 self._aux_cache_keys = set(
 190                         ["DEPEND", "EAPI", "HDEPEND",
 191                         "INHERITED", "IUSE", "KEYWORDS", "LICENSE",
 192                         "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository",
 193                         "RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"])
 194
 195                 self._aux_cache = {}
 196                 self._broken_ebuilds = set()
 197
 198         def _create_pregen_cache(self, tree):
 199                 conf = self.repositories.get_repo_for_location(tree)
 200                 cache = conf.get_pregenerated_cache(
 201                         self._known_keys, readonly=True)
 202                 if cache is not None:
 203                         try:
 204                                 cache.ec = self.repositories.get_repo_for_location(tree).eclass_db
 205                         except AttributeError:
 206                                 pass
 207
 208                         if not cache.complete_eclass_entries:
 209                                 warnings.warn(
 210                                         ("Repository '%s' used deprecated 'pms' cache format. "
 211                                         "Please migrate to 'md5-dict' format.") % (conf.name,),
 212                                         DeprecationWarning)
 213
 214                 return cache
 215
 216         def _init_cache_dirs(self):
 217                 """Create /var/cache/edb/dep and adjust permissions for the portage
 218                 group."""
 219
 220                 dirmode  = 0o2070
 221                 modemask =    0o2
 222
 223                 try:
 224                         ensure_dirs(self.depcachedir, gid=portage_gid,
 225                                 mode=dirmode, mask=modemask)
 226                 except PortageException:
 227                         pass
 228
 229         def close_caches(self):
 230                 if not hasattr(self, "auxdb"):
 231                         # unhandled exception thrown from constructor
 232                         return
 233                 for x in self.auxdb:
 234                         self.auxdb[x].sync()
 235                 self.auxdb.clear()
 236
 237         def flush_cache(self):
 238                 for x in self.auxdb.values():
 239                         x.sync()
 240
 241         def findLicensePath(self, license_name):
 242                 for x in reversed(self.porttrees):
 243                         license_path = os.path.join(x, "licenses", license_name)
 244                         if os.access(license_path, os.R_OK):
 245                                 return license_path
 246                 return None
 247
 248         def findname(self,mycpv, mytree = None, myrepo = None):
 249                 return self.findname2(mycpv, mytree, myrepo)[0]
 250
 251         def getRepositoryPath(self, repository_id):
 252                 """
 253                 This function is required for GLEP 42 compliance; given a valid repository ID
 254                 it must return a path to the repository
 255                 TreeMap = { id:path }
 256                 """
 257                 return self.treemap.get(repository_id)
 258
 259         def getRepositoryName(self, canonical_repo_path):
 260                 """
 261                 This is the inverse of getRepositoryPath().
 262                 @param canonical_repo_path: the canonical path of a repository, as
 263                         resolved by os.path.realpath()
 264                 @type canonical_repo_path: String
 265                 @return: The repo_name for the corresponding repository, or None
 266                         if the path does not correspond a known repository
 267                 @rtype: String or None
 268                 """
 269                 try:
 270                         return self.repositories.get_name_for_location(canonical_repo_path)
 271                 except KeyError:
 272                         return None
 273
 274         def getRepositories(self):
 275                 """
 276                 This function is required for GLEP 42 compliance; it will return a list of
 277                 repository IDs
 278                 TreeMap = {id: path}
 279                 """
 280                 return self._ordered_repo_name_list
 281
 282         def getMissingRepoNames(self):
 283                 """
 284                 Returns a list of repository paths that lack profiles/repo_name.
 285                 """
 286                 return self.settings.repositories.missing_repo_names
 287
 288         def getIgnoredRepos(self):
 289                 """
 290                 Returns a list of repository paths that have been ignored, because
 291                 another repo with the same name exists.
 292                 """
 293                 return self.settings.repositories.ignored_repos
 294
 295         def findname2(self, mycpv, mytree=None, myrepo = None):
 296                 """
 297                 Returns the location of the CPV, and what overlay it was in.
 298                 Searches overlays first, then PORTDIR; this allows us to return the first
 299                 matching file.  As opposed to starting in portdir and then doing overlays
 300                 second, we would have to exhaustively search the overlays until we found
 301                 the file we wanted.
 302                 If myrepo is not None it will find packages from this repository(overlay)
 303                 """
 304                 if not mycpv:
 305                         return (None, 0)
 306
 307                 if myrepo is not None:
 308                         mytree = self.treemap.get(myrepo)
 309                         if mytree is None:
 310                                 return (None, 0)
 311
 312                 mysplit = mycpv.split("/")
 313                 psplit = pkgsplit(mysplit[1])
 314                 if psplit is None or len(mysplit) != 2:
 315                         raise InvalidPackageName(mycpv)
 316
 317                 # For optimal performace in this hot spot, we do manual unicode
 318                 # handling here instead of using the wrapped os module.
 319                 encoding = _encodings['fs']
 320                 errors = 'strict'
 321
 322                 if mytree:
 323                         mytrees = [mytree]
 324                 else:
 325                         mytrees = reversed(self.porttrees)
 326
 327                 relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \
 328                         mysplit[1] + ".ebuild"
 329
 330                 for x in mytrees:
 331                         filename = x + _os.sep + relative_path
 332                         if _os.access(_unicode_encode(filename,
 333                                 encoding=encoding, errors=errors), _os.R_OK):
 334                                 return (filename, x)
 335                 return (None, 0)
 336
 337         def _write_cache(self, cpv, repo_path, metadata, ebuild_hash):
 338
 339                 try:
 340                         cache = self.auxdb[repo_path]
 341                         chf = cache.validation_chf
 342                         metadata['_%s_' % chf] = getattr(ebuild_hash, chf)
 343                 except CacheError:
 344                         # Normally this shouldn't happen, so we'll show
 345                         # a traceback for debugging purposes.
 346                         traceback.print_exc()
 347                         cache = None
 348
 349                 if cache is not None:
 350                         try:
 351                                 cache[cpv] = metadata
 352                         except CacheError:
 353                                 # Normally this shouldn't happen, so we'll show
 354                                 # a traceback for debugging purposes.
 355                                 traceback.print_exc()
 356
 357         def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
 358                 try:
 359                         ebuild_hash = eclass_cache.hashed_path(ebuild_path)
 360                         # snag mtime since we use it later, and to trigger stat failure
 361                         # if it doesn't exist
 362                         ebuild_hash.mtime
 363                 except FileNotFound:
 364                         writemsg(_("!!! aux_get(): ebuild for " \
 365                                 "'%s' does not exist at:\n") % (cpv,), noiselevel=-1)
 366                         writemsg("!!!            %s\n" % ebuild_path, noiselevel=-1)
 367                         raise KeyError(cpv)
 368
 369                 # Pull pre-generated metadata from the metadata/cache/
 370                 # directory if it exists and is valid, otherwise fall
 371                 # back to the normal writable cache.
 372                 auxdbs = []
 373                 pregen_auxdb = self._pregen_auxdb.get(repo_path)
 374                 if pregen_auxdb is not None:
 375                         auxdbs.append(pregen_auxdb)
 376                 ro_auxdb = self._ro_auxdb.get(repo_path)
 377                 if ro_auxdb is not None:
 378                         auxdbs.append(ro_auxdb)
 379                 auxdbs.append(self.auxdb[repo_path])
 380                 eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db
 381
 382                 for auxdb in auxdbs:
 383                         try:
 384                                 metadata = auxdb[cpv]
 385                         except KeyError:
 386                                 continue
 387                         except CacheError:
 388                                 if not auxdb.readonly:
 389                                         try:
 390                                                 del auxdb[cpv]
 391                                         except (KeyError, CacheError):
 392                                                 pass
 393                                 continue
 394                         eapi = metadata.get('EAPI', '').strip()
 395                         if not eapi:
 396                                 eapi = '0'
 397                                 metadata['EAPI'] = eapi
 398                         if not eapi_is_supported(eapi):
 399                                 # Since we're supposed to be able to efficiently obtain the
 400                                 # EAPI from _parse_eapi_ebuild_head, we disregard cache entries
 401                                 # for unsupported EAPIs.
 402                                 continue
 403                         if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
 404                                 break
 405                 else:
 406                         metadata = None
 407
 408                 return (metadata, ebuild_hash)
 409
 410         def aux_get(self, mycpv, mylist, mytree=None, myrepo=None):
 411                 "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
 412                 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
 413                 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
 414                 cache_me = False
 415                 if myrepo is not None:
 416                         mytree = self.treemap.get(myrepo)
 417                         if mytree is None:
 418                                 raise KeyError(myrepo)
 419
 420                 if mytree is not None and len(self.porttrees) == 1 \
 421                         and mytree == self.porttrees[0]:
 422                         # mytree matches our only tree, so it's safe to
 423                         # ignore mytree and cache the result
 424                         mytree = None
 425                         myrepo = None
 426
 427                 if mytree is None:
 428                         cache_me = True
 429                 if mytree is None and not self._known_keys.intersection(
 430                         mylist).difference(self._aux_cache_keys):
 431                         aux_cache = self._aux_cache.get(mycpv)
 432                         if aux_cache is not None:
 433                                 return [aux_cache.get(x, "") for x in mylist]
 434                         cache_me = True
 435
 436                 try:
 437                         cat, pkg = mycpv.split("/", 1)
 438                 except ValueError:
 439                         # Missing slash. Can't find ebuild so raise KeyError.
 440                         raise KeyError(mycpv)
 441
 442                 myebuild, mylocation = self.findname2(mycpv, mytree)
 443
 444                 if not myebuild:
 445                         writemsg("!!! aux_get(): %s\n" % \
 446                                 _("ebuild not found for '%s'") % mycpv, noiselevel=1)
 447                         raise KeyError(mycpv)
 448
 449                 mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
 450                 doregen = mydata is None
 451
 452                 if doregen:
 453                         if myebuild in self._broken_ebuilds:
 454                                 raise KeyError(mycpv)
 455
 456                         proc = EbuildMetadataPhase(cpv=mycpv,
 457                                 ebuild_hash=ebuild_hash, portdb=self,
 458                                 repo_path=mylocation, scheduler=self._event_loop,
 459                                 settings=self.doebuild_settings)
 460
 461                         proc.start()
 462                         proc.wait()
 463
 464                         if proc.returncode != os.EX_OK:
 465                                 self._broken_ebuilds.add(myebuild)
 466                                 raise KeyError(mycpv)
 467
 468                         mydata = proc.metadata
 469
 470                 mydata["repository"] = self.repositories.get_name_for_location(mylocation)
 471                 mydata["_mtime_"] = ebuild_hash.mtime
 472                 eapi = mydata.get("EAPI")
 473                 if not eapi:
 474                         eapi = "0"
 475                         mydata["EAPI"] = eapi
 476                 if eapi_is_supported(eapi):
 477                         mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", []))
 478
 479                 #finally, we look at our internal cache entry and return the requested data.
 480                 returnme = [mydata.get(x, "") for x in mylist]
 481
 482                 if cache_me:
 483                         aux_cache = {}
 484                         for x in self._aux_cache_keys:
 485                                 aux_cache[x] = mydata.get(x, "")
 486                         self._aux_cache[mycpv] = aux_cache
 487
 488                 return returnme
 489
 490         def getFetchMap(self, mypkg, useflags=None, mytree=None):
 491                 """
 492                 Get the SRC_URI metadata as a dict which maps each file name to a
 493                 set of alternative URIs.
 494
 495                 @param mypkg: cpv for an ebuild
 496                 @type mypkg: String
 497                 @param useflags: a collection of enabled USE flags, for evaluation of
 498                         conditionals
 499                 @type useflags: set, or None to enable all conditionals
 500                 @param mytree: The canonical path of the tree in which the ebuild
 501                         is located, or None for automatic lookup
 502                 @type mypkg: String
 503                 @return: A dict which maps each file name to a set of alternative
 504                         URIs.
 505                 @rtype: dict
 506                 """
 507
 508                 try:
 509                         eapi, myuris = self.aux_get(mypkg,
 510                                 ["EAPI", "SRC_URI"], mytree=mytree)
 511                 except KeyError:
 512                         # Convert this to an InvalidDependString exception since callers
 513                         # already handle it.
 514                         raise portage.exception.InvalidDependString(
 515                                 "getFetchMap(): aux_get() error reading "+mypkg+"; aborting.")
 516
 517                 if not eapi_is_supported(eapi):
 518                         # Convert this to an InvalidDependString exception
 519                         # since callers already handle it.
 520                         raise portage.exception.InvalidDependString(
 521                                 "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \
 522                                 (mypkg, eapi))
 523
 524                 return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris},
 525                         use=useflags)
 526
 527         def getfetchsizes(self, mypkg, useflags=None, debug=0, myrepo=None):
 528                 # returns a filename:size dictionnary of remaining downloads
 529                 myebuild, mytree = self.findname2(mypkg, myrepo=myrepo)
 530                 if myebuild is None:
 531                         raise AssertionError(_("ebuild not found for '%s'") % mypkg)
 532                 pkgdir = os.path.dirname(myebuild)
 533                 mf = self.repositories.get_repo_for_location(
 534                         os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
 535                                 pkgdir, self.settings["DISTDIR"])
 536                 checksums = mf.getDigests()
 537                 if not checksums:
 538                         if debug:
 539                                 writemsg(_("[empty/missing/bad digest]: %s\n") % (mypkg,))
 540                         return {}
 541                 filesdict={}
 542                 myfiles = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree)
 543                 #XXX: maybe this should be improved: take partial downloads
 544                 # into account? check checksums?
 545                 for myfile in myfiles:
 546                         try:
 547                                 fetch_size = int(checksums[myfile]["size"])
 548                         except (KeyError, ValueError):
 549                                 if debug:
 550                                         writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg})
 551                                 continue
 552                         file_path = os.path.join(self.settings["DISTDIR"], myfile)
 553                         mystat = None
 554                         try:
 555                                 mystat = os.stat(file_path)
 556                         except OSError:
 557                                 pass
 558                         if mystat is None:
 559                                 existing_size = 0
 560                                 ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS")
 561                                 if ro_distdirs is not None:
 562                                         for x in shlex_split(ro_distdirs):
 563                                                 try:
 564                                                         mystat = os.stat(os.path.join(x, myfile))
 565                                                 except OSError:
 566                                                         pass
 567                                                 else:
 568                                                         if mystat.st_size == fetch_size:
 569                                                                 existing_size = fetch_size
 570                                                                 break
 571                         else:
 572                                 existing_size = mystat.st_size
 573                         remaining_size = fetch_size - existing_size
 574                         if remaining_size > 0:
 575                                 # Assume the download is resumable.
 576                                 filesdict[myfile] = remaining_size
 577                         elif remaining_size < 0:
 578                                 # The existing file is too large and therefore corrupt.
 579                                 filesdict[myfile] = int(checksums[myfile]["size"])
 580                 return filesdict
 581
 582         def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False, myrepo=None):
 583                 """
 584                 TODO: account for PORTAGE_RO_DISTDIRS
 585                 """
 586                 if all:
 587                         useflags = None
 588                 elif useflags is None:
 589                         if mysettings:
 590                                 useflags = mysettings["USE"].split()
 591                 if myrepo is not None:
 592                         mytree = self.treemap.get(myrepo)
 593                         if mytree is None:
 594                                 return False
 595                 else:
 596                         mytree = None
 597
 598                 myfiles = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree)
 599                 myebuild = self.findname(mypkg, myrepo=myrepo)
 600                 if myebuild is None:
 601                         raise AssertionError(_("ebuild not found for '%s'") % mypkg)
 602                 pkgdir = os.path.dirname(myebuild)
 603                 mf = self.repositories.get_repo_for_location(
 604                         os.path.dirname(os.path.dirname(pkgdir)))
 605                 mf = mf.load_manifest(pkgdir, self.settings["DISTDIR"])
 606                 mysums = mf.getDigests()
 607
 608                 failures = {}
 609                 for x in myfiles:
 610                         if not mysums or x not in mysums:
 611                                 ok     = False
 612                                 reason = _("digest missing")
 613                         else:
 614                                 try:
 615                                         ok, reason = portage.checksum.verify_all(
 616                                                 os.path.join(self.settings["DISTDIR"], x), mysums[x])
 617                                 except FileNotFound as e:
 618                                         ok = False
 619                                         reason = _("File Not Found: '%s'") % (e,)
 620                         if not ok:
 621                                 failures[x] = reason
 622                 if failures:
 623                         return False
 624                 return True
 625
 626         def cpv_exists(self, mykey, myrepo=None):
 627                 "Tells us whether an actual ebuild exists on disk (no masking)"
 628                 cps2 = mykey.split("/")
 629                 cps = catpkgsplit(mykey, silent=0)
 630                 if not cps:
 631                         #invalid cat/pkg-v
 632                         return 0
 633                 if self.findname(cps[0] + "/" + cps2[1], myrepo=myrepo):
 634                         return 1
 635                 else:
 636                         return 0
 637
 638         def cp_all(self, categories=None, trees=None):
 639                 """
 640                 This returns a list of all keys in our tree or trees
 641                 @param categories: optional list of categories to search or
 642                         defaults to self.settings.categories
 643                 @param trees: optional list of trees to search the categories in or
 644                         defaults to self.porttrees
 645                 @rtype list of [cat/pkg,...]
 646                 """
 647                 d = {}
 648                 if categories is None:
 649                         categories = self.settings.categories
 650                 if trees is None:
 651                         trees = self.porttrees
 652                 for x in categories:
 653                         for oroot in trees:
 654                                 for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1):
 655                                         try:
 656                                                 atom = Atom("%s/%s" % (x, y))
 657                                         except InvalidAtom:
 658                                                 continue
 659                                         if atom != atom.cp:
 660                                                 continue
 661                                         d[atom.cp] = None
 662                 l = list(d)
 663                 l.sort()
 664                 return l
 665
 666         def cp_list(self, mycp, use_cache=1, mytree=None):
 667                 # NOTE: Cache can be safely shared with the match cache, since the
 668                 # match cache uses the result from dep_expand for the cache_key.
 669                 if self.frozen and mytree is not None \
 670                         and len(self.porttrees) == 1 \
 671                         and mytree == self.porttrees[0]:
 672                         # mytree matches our only tree, so it's safe to
 673                         # ignore mytree and cache the result
 674                         mytree = None
 675
 676                 if self.frozen and mytree is None:
 677                         cachelist = self.xcache["cp-list"].get(mycp)
 678                         if cachelist is not None:
 679                                 # Try to propagate this to the match-all cache here for
 680                                 # repoman since he uses separate match-all caches for each
 681                                 # profile (due to differences in _get_implicit_iuse).
 682                                 self.xcache["match-all"][(mycp, mycp)] = cachelist
 683                                 return cachelist[:]
 684                 mysplit = mycp.split("/")
 685                 invalid_category = mysplit[0] not in self._categories
 686                 d={}
 687                 if mytree is not None:
 688                         if isinstance(mytree, basestring):
 689                                 mytrees = [mytree]
 690                         else:
 691                                 # assume it's iterable
 692                                 mytrees = mytree
 693                 else:
 694                         mytrees = self.porttrees
 695                 for oroot in mytrees:
 696                         try:
 697                                 file_list = os.listdir(os.path.join(oroot, mycp))
 698                         except OSError:
 699                                 continue
 700                         for x in file_list:
 701                                 pf = None
 702                                 if x[-7:] == '.ebuild':
 703                                         pf = x[:-7]
 704
 705                                 if pf is not None:
 706                                         ps = pkgsplit(pf)
 707                                         if not ps:
 708                                                 writemsg(_("\nInvalid ebuild name: %s\n") % \
 709                                                         os.path.join(oroot, mycp, x), noiselevel=-1)
 710                                                 continue
 711                                         if ps[0] != mysplit[1]:
 712                                                 writemsg(_("\nInvalid ebuild name: %s\n") % \
 713                                                         os.path.join(oroot, mycp, x), noiselevel=-1)
 714                                                 continue
 715                                         ver_match = ver_regexp.match("-".join(ps[1:]))
 716                                         if ver_match is None or not ver_match.groups():
 717                                                 writemsg(_("\nInvalid ebuild version: %s\n") % \
 718                                                         os.path.join(oroot, mycp, x), noiselevel=-1)
 719                                                 continue
 720                                         d[_pkg_str(mysplit[0]+"/"+pf)] = None
 721                 if invalid_category and d:
 722                         writemsg(_("\n!!! '%s' has a category that is not listed in " \
 723                                 "%setc/portage/categories\n") % \
 724                                 (mycp, self.settings["PORTAGE_CONFIGROOT"]), noiselevel=-1)
 725                         mylist = []
 726                 else:
 727                         mylist = list(d)
 728                 # Always sort in ascending order here since it's handy
 729                 # and the result can be easily cached and reused.
 730                 self._cpv_sort_ascending(mylist)
 731                 if self.frozen and mytree is None:
 732                         cachelist = mylist[:]
 733                         self.xcache["cp-list"][mycp] = cachelist
 734                         self.xcache["match-all"][(mycp, mycp)] = cachelist
 735                 return mylist
 736
 737         def freeze(self):
 738                 for x in "bestmatch-visible", "cp-list", "match-all", \
 739                         "match-all-cpv-only", "match-visible", "minimum-all", \
 740                         "minimum-visible":
 741                         self.xcache[x]={}
 742                 self.frozen=1
 743
 744         def melt(self):
 745                 self.xcache = {}
 746                 self.frozen = 0
 747
 748         def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
 749                 "caching match function; very trick stuff"
 750                 if level == "list-visible":
 751                         level = "match-visible"
 752                         warnings.warn("The 'list-visible' mode of "
 753                                 "portage.dbapi.porttree.portdbapi.xmatch "
 754                                 "has been renamed to match-visible",
 755                                 DeprecationWarning, stacklevel=2)
 756
 757                 if mydep is None:
 758                         #this stuff only runs on first call of xmatch()
 759                         #create mydep, mykey from origdep
 760                         mydep = dep_expand(origdep, mydb=self, settings=self.settings)
 761                         mykey = mydep.cp
 762
 763                 #if no updates are being made to the tree, we can consult our xcache...
 764                 cache_key = None
 765                 if self.frozen:
 766                         cache_key = (mydep, mydep.unevaluated_atom)
 767                         try:
 768                                 return self.xcache[level][cache_key][:]
 769                         except KeyError:
 770                                 pass
 771
 772                 myval = None
 773                 mytree = None
 774                 if mydep.repo is not None:
 775                         mytree = self.treemap.get(mydep.repo)
 776                         if mytree is None:
 777                                 if level.startswith("match-"):
 778                                         myval = []
 779                                 else:
 780                                         myval = ""
 781
 782                 if myval is not None:
 783                         # Unknown repo, empty result.
 784                         pass
 785                 elif level == "match-all-cpv-only":
 786                         # match *all* packages, only against the cpv, in order
 787                         # to bypass unnecessary cache access for things like IUSE
 788                         # and SLOT.
 789                         if mydep == mykey:
 790                                 # Share cache with match-all/cp_list when the result is the
 791                                 # same. Note that this requires that mydep.repo is None and
 792                                 # thus mytree is also None.
 793                                 level = "match-all"
 794                                 myval = self.cp_list(mykey, mytree=mytree)
 795                         else:
 796                                 myval = match_from_list(mydep,
 797                                         self.cp_list(mykey, mytree=mytree))
 798
 799                 elif level in ("bestmatch-visible", "match-all", "match-visible",
 800                         "minimum-all", "minimum-visible"):
 801                         # Find the minimum matching visible version. This is optimized to
 802                         # minimize the number of metadata accesses (improves performance
 803                         # especially in cases where metadata needs to be generated).
 804                         if mydep == mykey:
 805                                 mylist = self.cp_list(mykey, mytree=mytree)
 806                         else:
 807                                 mylist = match_from_list(mydep,
 808                                         self.cp_list(mykey, mytree=mytree))
 809
 810                         visibility_filter = level not in ("match-all", "minimum-all")
 811                         single_match = level not in ("match-all", "match-visible")
 812                         myval = []
 813                         aux_keys = list(self._aux_cache_keys)
 814                         if level == "bestmatch-visible":
 815                                 iterfunc = reversed
 816                         else:
 817                                 iterfunc = iter
 818
 819                         if mydep.repo is not None:
 820                                 repos = [mydep.repo]
 821                         else:
 822                                 # We iterate over self.porttrees, since it's common to
 823                                 # tweak this attribute in order to adjust match behavior.
 824                                 repos = []
 825                                 for tree in reversed(self.porttrees):
 826                                         repos.append(self.repositories.get_name_for_location(tree))
 827
 828                         for cpv in iterfunc(mylist):
 829                                 for repo in repos:
 830                                         try:
 831                                                 metadata = dict(zip(aux_keys,
 832                                                         self.aux_get(cpv, aux_keys, myrepo=repo)))
 833                                         except KeyError:
 834                                                 # ebuild not in this repo, or masked by corruption
 835                                                 continue
 836
 837                                         try:
 838                                                 pkg_str = _pkg_str(cpv, metadata=metadata,
 839                                                         settings=self.settings)
 840                                         except InvalidData:
 841                                                 continue
 842
 843                                         if visibility_filter and not self._visible(pkg_str, metadata):
 844                                                 continue
 845
 846                                         if mydep.slot is not None and \
 847                                                 not _match_slot(mydep, pkg_str):
 848                                                 continue
 849
 850                                         if mydep.unevaluated_atom.use is not None and \
 851                                                 not self._match_use(mydep, pkg_str, metadata):
 852                                                 continue
 853
 854                                         myval.append(pkg_str)
 855                                         # only yield a given cpv once
 856                                         break
 857
 858                                 if myval and single_match:
 859                                         break
 860
 861                         if single_match:
 862                                 if myval:
 863                                         myval = myval[0]
 864                                 else:
 865                                         myval = ""
 866
 867                 elif level == "bestmatch-list":
 868                         #dep match -- find best match but restrict search to sublist
 869                         warnings.warn("The 'bestmatch-list' mode of "
 870                                 "portage.dbapi.porttree.portdbapi.xmatch is deprecated",
 871                                 DeprecationWarning, stacklevel=2)
 872                         myval = best(list(self._iter_match(mydep, mylist)))
 873                 elif level == "match-list":
 874                         #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible())
 875                         warnings.warn("The 'match-list' mode of "
 876                                 "portage.dbapi.porttree.portdbapi.xmatch is deprecated",
 877                                 DeprecationWarning, stacklevel=2)
 878                         myval = list(self._iter_match(mydep, mylist))
 879                 else:
 880                         raise AssertionError(
 881                                 "Invalid level argument: '%s'" % level)
 882
 883                 if self.frozen:
 884                         xcache_this_level = self.xcache.get(level)
 885                         if xcache_this_level is not None:
 886                                 xcache_this_level[cache_key] = myval
 887                                 if not isinstance(myval, _pkg_str):
 888                                         myval = myval[:]
 889
 890                 return myval
 891
 892         def match(self, mydep, use_cache=1):
 893                 return self.xmatch("match-visible", mydep)
 894
 895         def gvisible(self, mylist):
 896                 warnings.warn("The 'gvisible' method of "
 897                         "portage.dbapi.porttree.portdbapi "
 898                         "is deprecated",
 899                         DeprecationWarning, stacklevel=2)
 900                 return list(self._iter_visible(iter(mylist)))
 901
 902         def visible(self, cpv_iter):
 903                 warnings.warn("The 'visible' method of "
 904                         "portage.dbapi.porttree.portdbapi "
 905                         "is deprecated",
 906                         DeprecationWarning, stacklevel=2)
 907                 if cpv_iter is None:
 908                         return []
 909                 return list(self._iter_visible(iter(cpv_iter)))
 910
 911         def _iter_visible(self, cpv_iter, myrepo=None):
 912                 """
 913                 Return a new list containing only visible packages.
 914                 """
 915                 aux_keys = list(self._aux_cache_keys)
 916                 metadata = {}
 917
 918                 if myrepo is not None:
 919                         repos = [myrepo]
 920                 else:
 921                         # We iterate over self.porttrees, since it's common to
 922                         # tweak this attribute in order to adjust match behavior.
 923                         repos = []
 924                         for tree in reversed(self.porttrees):
 925                                 repos.append(self.repositories.get_name_for_location(tree))
 926
 927                 for mycpv in cpv_iter:
 928                         for repo in repos:
 929                                 metadata.clear()
 930                                 try:
 931                                         metadata.update(zip(aux_keys,
 932                                                 self.aux_get(mycpv, aux_keys, myrepo=repo)))
 933                                 except KeyError:
 934                                         continue
 935                                 except PortageException as e:
 936                                         writemsg("!!! Error: aux_get('%s', %s)\n" %
 937                                                 (mycpv, aux_keys), noiselevel=-1)
 938                                         writemsg("!!! %s\n" % (e,), noiselevel=-1)
 939                                         del e
 940                                         continue
 941
 942                                 if not self._visible(mycpv, metadata):
 943                                         continue
 944
 945                                 yield mycpv
 946                                 # only yield a given cpv once
 947                                 break
 948
 949         def _visible(self, cpv, metadata):
 950                 eapi = metadata["EAPI"]
 951                 if not eapi_is_supported(eapi):
 952                         return False
 953                 if _eapi_is_deprecated(eapi):
 954                         return False
 955                 if not metadata["SLOT"]:
 956                         return False
 957
 958                 settings = self.settings
 959                 if settings._getMaskAtom(cpv, metadata):
 960                         return False
 961                 if settings._getMissingKeywords(cpv, metadata):
 962                         return False
 963                 if settings.local_config:
 964                         metadata['CHOST'] = settings.get('CHOST', '')
 965                         if not settings._accept_chost(cpv, metadata):
 966                                 return False
 967                         metadata["USE"] = ""
 968                         if "?" in metadata["LICENSE"] or \
 969                                 "?" in metadata["PROPERTIES"]:
 970                                 self.doebuild_settings.setcpv(cpv, mydb=metadata)
 971                                 metadata['USE'] = self.doebuild_settings['PORTAGE_USE']
 972                         try:
 973                                 if settings._getMissingLicenses(cpv, metadata):
 974                                         return False
 975                                 if settings._getMissingProperties(cpv, metadata):
 976                                         return False
 977                         except InvalidDependString:
 978                                 return False
 979
 980                 return True
 981
 982 def close_portdbapi_caches():
 983         for i in portdbapi.portdbapi_instances:
 984                 i.close_caches()
 985
 986 portage.process.atexit_register(portage.portageexit)
 987
 988 class portagetree(object):
 989         def __init__(self, root=None, virtual=DeprecationWarning, settings=None):
 990                 """
 991                 Constructor for a PortageTree
 992
 993                 @param root: deprecated, defaults to settings['ROOT']
 994                 @type root: String/Path
 995                 @param virtual: UNUSED
 996                 @type virtual: No Idea
 997                 @param settings: Portage Configuration object (portage.settings)
 998                 @type settings: Instance of portage.config
 999                 """
1000
1001                 if settings is None:
1002                         settings = portage.settings
1003                 self.settings = settings
1004
1005                 if root is not None and root != settings['ROOT']:
1006                         warnings.warn("The root parameter of the " + \
1007                                 "portage.dbapi.porttree.portagetree" + \
1008                                 " constructor is now unused. Use " + \
1009                                 "settings['ROOT'] instead.",
1010                                 DeprecationWarning, stacklevel=2)
1011
1012                 if virtual is not DeprecationWarning:
1013                         warnings.warn("The 'virtual' parameter of the "
1014                                 "portage.dbapi.porttree.portagetree"
1015                                 " constructor is unused",
1016                                 DeprecationWarning, stacklevel=2)
1017
1018                 self.portroot = settings["PORTDIR"]
1019                 self.__virtual = virtual
1020                 self.dbapi = portdbapi(mysettings=settings)
1021
1022         @property
1023         def root(self):
1024                 warnings.warn("The root attribute of " + \
1025                         "portage.dbapi.porttree.portagetree" + \
1026                         " is deprecated. Use " + \
1027                         "settings['ROOT'] instead.",
1028                         DeprecationWarning, stacklevel=3)
1029                 return self.settings['ROOT']
1030
1031         @property
1032         def virtual(self):
1033                 warnings.warn("The 'virtual' attribute of " + \
1034                         "portage.dbapi.porttree.portagetree" + \
1035                         " is deprecated.",
1036                         DeprecationWarning, stacklevel=3)
1037                 return self.__virtual
1038
1039         def dep_bestmatch(self,mydep):
1040                 "compatibility method"
1041                 mymatch = self.dbapi.xmatch("bestmatch-visible",mydep)
1042                 if mymatch is None:
1043                         return ""
1044                 return mymatch
1045
1046         def dep_match(self,mydep):
1047                 "compatibility method"
1048                 mymatch = self.dbapi.xmatch("match-visible",mydep)
1049                 if mymatch is None:
1050                         return []
1051                 return mymatch
1052
1053         def exists_specific(self,cpv):
1054                 return self.dbapi.cpv_exists(cpv)
1055
1056         def getallnodes(self):
1057                 """new behavior: these are all *unmasked* nodes.  There may or may not be available
1058                 masked package for nodes in this nodes list."""
1059                 return self.dbapi.cp_all()
1060
1061         def getname(self, pkgname):
1062                 "returns file location for this particular package (DEPRECATED)"
1063                 if not pkgname:
1064                         return ""
1065                 mysplit = pkgname.split("/")
1066                 psplit = pkgsplit(mysplit[1])
1067                 return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild"
1068
1069         def getslot(self,mycatpkg):
1070                 "Get a slot for a catpkg; assume it exists."
1071                 myslot = ""
1072                 try:
1073                         myslot = self.dbapi._pkg_str(mycatpkg, None).slot
1074                 except KeyError:
1075                         pass
1076                 return myslot
1077
1078 class FetchlistDict(Mapping):
1079         """
1080         This provide a mapping interface to retrieve fetch lists. It's used
1081         to allow portage.manifest.Manifest to access fetch lists via a standard
1082         mapping interface rather than use the dbapi directly.
1083         """
1084         def __init__(self, pkgdir, settings, mydbapi):
1085                 """pkgdir is a directory containing ebuilds and settings is passed into
1086                 portdbapi.getfetchlist for __getitem__ calls."""
1087                 self.pkgdir = pkgdir
1088                 self.cp = os.sep.join(pkgdir.split(os.sep)[-2:])
1089                 self.settings = settings
1090                 self.mytree = os.path.realpath(os.path.dirname(os.path.dirname(pkgdir)))
1091                 self.portdb = mydbapi
1092
1093         def __getitem__(self, pkg_key):
1094                 """Returns the complete fetch list for a given package."""
1095                 return list(self.portdb.getFetchMap(pkg_key, mytree=self.mytree))
1096
1097         def __contains__(self, cpv):
1098                 return cpv in self.__iter__()
1099
1100         def has_key(self, pkg_key):
1101                 """Returns true if the given package exists within pkgdir."""
1102                 warnings.warn("portage.dbapi.porttree.FetchlistDict.has_key() is "
1103                         "deprecated, use the 'in' operator instead",
1104                         DeprecationWarning, stacklevel=2)
1105                 return pkg_key in self
1106
1107         def __iter__(self):
1108                 return iter(self.portdb.cp_list(self.cp, mytree=self.mytree))
1109
1110         def __len__(self):
1111                 """This needs to be implemented in order to avoid
1112                 infinite recursion in some cases."""
1113                 return len(self.portdb.cp_list(self.cp, mytree=self.mytree))
1114
1115         def keys(self):
1116                 """Returns keys for all packages within pkgdir"""
1117                 return self.portdb.cp_list(self.cp, mytree=self.mytree)
1118
1119         if sys.hexversion >= 0x3000000:
1120                 keys = __iter__
1121
1122 def _parse_uri_map(cpv, metadata, use=None):
1123
1124         myuris = use_reduce(metadata.get('SRC_URI', ''),
1125                 uselist=use, matchall=(use is None),
1126                 is_src_uri=True,
1127                 eapi=metadata['EAPI'])
1128
1129         uri_map = OrderedDict()
1130
1131         myuris.reverse()
1132         while myuris:
1133                 uri = myuris.pop()
1134                 if myuris and myuris[-1] == "->":
1135                         myuris.pop()
1136                         distfile = myuris.pop()
1137                 else:
1138                         distfile = os.path.basename(uri)
1139                         if not distfile:
1140                                 raise portage.exception.InvalidDependString(
1141                                         ("getFetchMap(): '%s' SRC_URI has no file " + \
1142                                         "name: '%s'") % (cpv, uri))
1143
1144                 uri_set = uri_map.get(distfile)
1145                 if uri_set is None:
1146                         # Use OrderedDict to preserve order from SRC_URI
1147                         # while ensuring uniqueness.
1148                         uri_set = OrderedDict()
1149                         uri_map[distfile] = uri_set
1150                 uri_set[uri] = True
1151
1152         # Convert OrderedDicts to tuples.
1153         for k, v in uri_map.items():
1154                 uri_map[k] = tuple(v)
1155
1156         return uri_map