pym/portage/dbapi/porttree.py

   1 # Copyright 1998-2007 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3 # $Id$
   4
   5 from portage.cache.cache_errors import CacheError
   6 from portage.const import REPO_NAME_LOC
   7 from portage.data import portage_gid, secpass
   8 from portage.dbapi import dbapi
   9 from portage.dep import use_reduce, paren_reduce, dep_getslot, dep_getkey, \
  10         match_from_list, match_to_list
  11 from portage.exception import OperationNotPermitted, PortageException, \
  12         UntrustedSignature, SecurityViolation, InvalidSignature, MissingSignature, \
  13         FileNotFound, InvalidDependString
  14 from portage.manifest import Manifest
  15 from portage.output import red
  16 from portage.util import ensure_dirs, writemsg, apply_recursive_permissions
  17 from portage.versions import pkgsplit, catpkgsplit, best
  18
  19 import portage.gpg, portage.checksum
  20
  21 from portage import eclass_cache, auxdbkeys, auxdbkeylen, doebuild, flatten, \
  22         listdir, dep_expand, eapi_is_supported, key_expand, dep_check
  23
  24 import os, stat, sys
  25
  26
  27 class portdbapi(dbapi):
  28         """this tree will scan a portage directory located at root (passed to init)"""
  29         portdbapi_instances = []
  30
  31         def __init__(self, porttree_root, mysettings=None):
  32                 portdbapi.portdbapi_instances.append(self)
  33
  34                 from portage import config
  35                 if mysettings:
  36                         self.mysettings = mysettings
  37                 else:
  38                         from portage import settings
  39                         self.mysettings = config(clone=settings)
  40                 self._categories = set(self.mysettings.categories)
  41                 # This is strictly for use in aux_get() doebuild calls when metadata
  42                 # is generated by the depend phase.  It's safest to use a clone for
  43                 # this purpose because doebuild makes many changes to the config
  44                 # instance that is passed in.
  45                 self.doebuild_settings = config(clone=self.mysettings)
  46
  47                 self.manifestVerifyLevel = None
  48                 self.manifestVerifier = None
  49                 self.manifestCache = {}    # {location: [stat, md5]}
  50                 self.manifestMissingCache = []
  51
  52                 if "gpg" in self.mysettings.features:
  53                         self.manifestVerifyLevel = portage.gpg.EXISTS
  54                         if "strict" in self.mysettings.features:
  55                                 self.manifestVerifyLevel = portage.gpg.MARGINAL
  56                                 self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel)
  57                         elif "severe" in self.mysettings.features:
  58                                 self.manifestVerifyLevel = portage.gpg.TRUSTED
  59                                 self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", requireSignedRing=True, minimumTrust=self.manifestVerifyLevel)
  60                         else:
  61                                 self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel)
  62
  63                 #self.root=settings["PORTDIR"]
  64                 self.porttree_root = os.path.realpath(porttree_root)
  65
  66                 self.depcachedir = self.mysettings.depcachedir[:]
  67
  68                 self.tmpfs = self.mysettings["PORTAGE_TMPFS"]
  69                 if self.tmpfs and not os.path.exists(self.tmpfs):
  70                         self.tmpfs = None
  71                 if self.tmpfs and not os.access(self.tmpfs, os.W_OK):
  72                         self.tmpfs = None
  73                 if self.tmpfs and not os.access(self.tmpfs, os.R_OK):
  74                         self.tmpfs = None
  75
  76                 self.eclassdb = eclass_cache.cache(self.porttree_root,
  77                         overlays=self.mysettings["PORTDIR_OVERLAY"].split())
  78
  79                 self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule")
  80
  81                 #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
  82                 self.xcache = {}
  83                 self.frozen = 0
  84
  85                 self.porttrees = [self.porttree_root] + \
  86                         [os.path.realpath(t) for t in self.mysettings["PORTDIR_OVERLAY"].split()]
  87                 self.treemap = {}
  88                 for path in self.porttrees:
  89                         repo_name_path = os.path.join(path, REPO_NAME_LOC)
  90                         try:
  91                                 repo_name = open(repo_name_path, 'r').readline().strip()
  92                                 self.treemap[repo_name] = path
  93                         except (OSError,IOError):
  94                                 # warn about missing repo_name at some other time, since we
  95                                 # don't want to see a warning every time the portage module is
  96                                 # imported.
  97                                 pass
  98
  99                 self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule")
 100                 self.auxdb = {}
 101                 self._init_cache_dirs()
 102                 # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
 103                 # ~harring
 104                 filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys)
 105                 if secpass < 1:
 106                         from portage.cache import metadata_overlay, volatile
 107                         for x in self.porttrees:
 108                                 db_ro = self.auxdbmodule(self.depcachedir, x,
 109                                         filtered_auxdbkeys, gid=portage_gid, readonly=True)
 110                                 self.auxdb[x] = metadata_overlay.database(
 111                                         self.depcachedir, x, filtered_auxdbkeys,
 112                                         gid=portage_gid, db_rw=volatile.database,
 113                                         db_ro=db_ro)
 114                 else:
 115                         for x in self.porttrees:
 116                                 # location, label, auxdbkeys
 117                                 self.auxdb[x] = self.auxdbmodule(
 118                                         self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid)
 119                 # Selectively cache metadata in order to optimize dep matching.
 120                 self._aux_cache_keys = set(["EAPI", "KEYWORDS", "LICENSE", "SLOT"])
 121                 self._aux_cache = {}
 122                 self._broken_ebuilds = set()
 123
 124         def _init_cache_dirs(self):
 125                 """Create /var/cache/edb/dep and adjust permissions for the portage
 126                 group."""
 127
 128                 dirmode = 02070
 129                 filemode =   060
 130                 modemask =    02
 131
 132                 try:
 133                         ensure_dirs(self.depcachedir, gid=portage_gid,
 134                                 mode=dirmode, mask=modemask)
 135                 except PortageException, e:
 136                         pass
 137
 138         def close_caches(self):
 139                 if not hasattr(self, "auxdb"):
 140                         # unhandled exception thrown from constructor
 141                         return
 142                 for x in self.auxdb:
 143                         self.auxdb[x].sync()
 144                 self.auxdb.clear()
 145
 146         def flush_cache(self):
 147                 for x in self.auxdb.values():
 148                         x.sync()
 149
 150         def finddigest(self,mycpv):
 151                 try:
 152                         mydig = self.findname2(mycpv)[0]
 153                         if not mydig:
 154                                 return ""
 155                         mydigs = mydig.split("/")[:-1]
 156                         mydig = "/".join(mydigs)
 157                         mysplit = mycpv.split("/")
 158                 except OSError:
 159                         return ""
 160                 return mydig+"/files/digest-"+mysplit[-1]
 161
 162         def findLicensePath(self, license_name):
 163                 mytrees = self.porttrees[:]
 164                 mytrees.reverse()
 165                 for x in mytrees:
 166                         license_path = os.path.join(x, "licenses", license_name)
 167                         if os.access(license_path, os.R_OK):
 168                                 return license_path
 169                 return None
 170
 171         def findname(self,mycpv):
 172                 return self.findname2(mycpv)[0]
 173
 174         def getRepositoryPath(self, repository_id):
 175                 """
 176                 This function is required for GLEP 42 compliance; given a valid repository ID
 177                 it must return a path to the repository
 178                 TreeMap = { id:path }
 179                 """
 180                 if repository_id in self.treemap:
 181                         return self.treemap[repository_id]
 182                 return None
 183
 184         def getRepositories(self):
 185                 """
 186                 This function is required for GLEP 42 compliance; it will return a list of
 187                 repository ID's
 188                 TreeMap = {id: path}
 189                 """
 190                 return [k for k in self.treemap if k]
 191
 192         def findname2(self, mycpv, mytree=None):
 193                 """
 194                 Returns the location of the CPV, and what overlay it was in.
 195                 Searches overlays first, then PORTDIR; this allows us to return the first
 196                 matching file.  As opposed to starting in portdir and then doing overlays
 197                 second, we would have to exhaustively search the overlays until we found
 198                 the file we wanted.
 199                 """
 200                 if not mycpv:
 201                         return "",0
 202                 mysplit = mycpv.split("/")
 203                 psplit = pkgsplit(mysplit[1])
 204
 205                 if mytree:
 206                         mytrees = [mytree]
 207                 else:
 208                         mytrees = self.porttrees[:]
 209                         mytrees.reverse()
 210                 if psplit:
 211                         for x in mytrees:
 212                                 file=x+"/"+mysplit[0]+"/"+psplit[0]+"/"+mysplit[1]+".ebuild"
 213                                 if os.access(file, os.R_OK):
 214                                         return[file, x]
 215                 return None, 0
 216
 217         def aux_get(self, mycpv, mylist, mytree=None):
 218                 "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
 219                 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
 220                 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
 221                 cache_me = False
 222                 if not mytree and not set(mylist).difference(self._aux_cache_keys):
 223                         aux_cache = self._aux_cache.get(mycpv)
 224                         if aux_cache is not None:
 225                                 return [aux_cache[x] for x in mylist]
 226                         cache_me = True
 227                 global auxdbkeys, auxdbkeylen
 228                 cat,pkg = mycpv.split("/", 1)
 229
 230                 myebuild, mylocation = self.findname2(mycpv, mytree)
 231
 232                 if not myebuild:
 233                         writemsg("!!! aux_get(): ebuild path for '%(cpv)s' not specified:\n" % {"cpv":mycpv},
 234                                 noiselevel=1)
 235                         writemsg("!!!            %s\n" % myebuild, noiselevel=1)
 236                         raise KeyError(mycpv)
 237
 238                 myManifestPath = "/".join(myebuild.split("/")[:-1])+"/Manifest"
 239                 if "gpg" in self.mysettings.features:
 240                         try:
 241                                 mys = portage.gpg.fileStats(myManifestPath)
 242                                 if (myManifestPath in self.manifestCache) and \
 243                                    (self.manifestCache[myManifestPath] == mys):
 244                                         pass
 245                                 elif self.manifestVerifier:
 246                                         if not self.manifestVerifier.verify(myManifestPath):
 247                                                 # Verification failed the desired level.
 248                                                 raise UntrustedSignature, "Untrusted Manifest: %(manifest)s" % {"manifest":myManifestPath}
 249
 250                                 if ("severe" in self.mysettings.features) and \
 251                                    (mys != portage.gpg.fileStats(myManifestPath)):
 252                                         raise SecurityViolation, "Manifest changed: %(manifest)s" % {"manifest":myManifestPath}
 253
 254                         except InvalidSignature, e:
 255                                 if ("strict" in self.mysettings.features) or \
 256                                    ("severe" in self.mysettings.features):
 257                                         raise
 258                                 writemsg("!!! INVALID MANIFEST SIGNATURE DETECTED: %(manifest)s\n" % {"manifest":myManifestPath})
 259                         except MissingSignature, e:
 260                                 if ("severe" in self.mysettings.features):
 261                                         raise
 262                                 if ("strict" in self.mysettings.features):
 263                                         if myManifestPath not in self.manifestMissingCache:
 264                                                 writemsg("!!! WARNING: Missing signature in: %(manifest)s\n" % {"manifest":myManifestPath})
 265                                                 self.manifestMissingCache.insert(0,myManifestPath)
 266                         except (OSError, FileNotFound), e:
 267                                 if ("strict" in self.mysettings.features) or \
 268                                    ("severe" in self.mysettings.features):
 269                                         raise SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)}
 270                                 writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath},
 271                                         noiselevel=-1)
 272
 273
 274                 if os.access(myebuild, os.R_OK):
 275                         emtime = os.stat(myebuild)[stat.ST_MTIME]
 276                 else:
 277                         writemsg("!!! aux_get(): ebuild for '%(cpv)s' does not exist at:\n" % {"cpv":mycpv},
 278                                 noiselevel=-1)
 279                         writemsg("!!!            %s\n" % myebuild,
 280                                 noiselevel=-1)
 281                         raise KeyError
 282
 283                 try:
 284                         mydata = self.auxdb[mylocation][mycpv]
 285                         if emtime != long(mydata.get("_mtime_", 0)):
 286                                 doregen = True
 287                         elif len(mydata.get("_eclasses_", [])) > 0:
 288                                 doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"])
 289                         else:
 290                                 doregen = False
 291
 292                 except KeyError:
 293                         doregen = True
 294                 except CacheError:
 295                         doregen = True
 296                         try:
 297                                 del self.auxdb[mylocation][mycpv]
 298                         except KeyError:
 299                                 pass
 300
 301                 writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2)
 302
 303                 if doregen:
 304                         if myebuild in self._broken_ebuilds:
 305                                 raise KeyError(mycpv)
 306                         writemsg("doregen: %s %s\n" % (doregen, mycpv), 2)
 307                         writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n", 1)
 308
 309                         self.doebuild_settings.reset()
 310                         mydata = {}
 311                         myret = doebuild(myebuild, "depend",
 312                                 self.doebuild_settings["ROOT"], self.doebuild_settings,
 313                                 dbkey=mydata, tree="porttree", mydbapi=self)
 314                         if myret != os.EX_OK:
 315                                 self._broken_ebuilds.add(myebuild)
 316                                 raise KeyError(mycpv)
 317
 318                         if "EAPI" not in mydata or not mydata["EAPI"].strip():
 319                                 mydata["EAPI"] = "0"
 320
 321                         if not eapi_is_supported(mydata["EAPI"]):
 322                                 # if newer version, wipe everything and negate eapi
 323                                 eapi = mydata["EAPI"]
 324                                 mydata = {}
 325                                 map(lambda x: mydata.setdefault(x, ""), auxdbkeys)
 326                                 mydata["EAPI"] = "-"+eapi
 327
 328                         if mydata.get("INHERITED", False):
 329                                 mydata["_eclasses_"] = self.eclassdb.get_eclass_data(mydata["INHERITED"].split())
 330                         else:
 331                                 mydata["_eclasses_"] = {}
 332
 333                         del mydata["INHERITED"]
 334
 335                         mydata["_mtime_"] = emtime
 336
 337                         self.auxdb[mylocation][mycpv] = mydata
 338
 339                 if not mydata.setdefault("EAPI", "0"):
 340                         mydata["EAPI"] = "0"
 341
 342                 #finally, we look at our internal cache entry and return the requested data.
 343                 returnme = []
 344                 for x in mylist:
 345                         if x == "INHERITED":
 346                                 returnme.append(' '.join(mydata.get("_eclasses_", [])))
 347                         else:
 348                                 returnme.append(mydata.get(x,""))
 349
 350                 if cache_me:
 351                         aux_cache = {}
 352                         for x in self._aux_cache_keys:
 353                                 aux_cache[x] = mydata.get(x, "")
 354                         self._aux_cache[mycpv] = aux_cache
 355
 356                 return returnme
 357
 358         def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None):
 359                 if mysettings is None:
 360                         mysettings = self.mysettings
 361                 try:
 362                         myuris = self.aux_get(mypkg, ["SRC_URI"], mytree=mytree)[0]
 363                 except KeyError:
 364                         # Convert this to an InvalidDependString exception since callers
 365                         # already handle it.
 366                         raise portage.exception.InvalidDependString(
 367                                 "getfetchlist(): aux_get() error reading "+mypkg+"; aborting.")
 368
 369                 if useflags is None:
 370                         useflags = mysettings["USE"].split()
 371
 372                 myurilist = paren_reduce(myuris)
 373                 myurilist = use_reduce(myurilist, uselist=useflags, matchall=all)
 374                 newuris = flatten(myurilist)
 375
 376                 myfiles = []
 377                 for x in newuris:
 378                         mya = os.path.basename(x)
 379                         if not mya:
 380                                 raise portage.exception.InvalidDependString("URI has no basename: '%s'" % x)
 381                         if not mya in myfiles:
 382                                 myfiles.append(mya)
 383                 return [newuris, myfiles]
 384
 385         def getfetchsizes(self, mypkg, useflags=None, debug=0):
 386                 # returns a filename:size dictionnary of remaining downloads
 387                 myebuild = self.findname(mypkg)
 388                 pkgdir = os.path.dirname(myebuild)
 389                 mf = Manifest(pkgdir, self.mysettings["DISTDIR"])
 390                 checksums = mf.getDigests()
 391                 if not checksums:
 392                         if debug:
 393                                 print "[empty/missing/bad digest]: "+mypkg
 394                         return None
 395                 filesdict={}
 396                 if useflags is None:
 397                         myuris, myfiles = self.getfetchlist(mypkg,all=1)
 398                 else:
 399                         myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags)
 400                 #XXX: maybe this should be improved: take partial downloads
 401                 # into account? check checksums?
 402                 for myfile in myfiles:
 403                         if myfile not in checksums:
 404                                 if debug:
 405                                         writemsg("[bad digest]: missing %s for %s\n" % (myfile, mypkg))
 406                                 continue
 407                         file_path = os.path.join(self.mysettings["DISTDIR"], myfile)
 408                         mystat = None
 409                         try:
 410                                 mystat = os.stat(file_path)
 411                         except OSError, e:
 412                                 pass
 413                         if mystat is None:
 414                                 existing_size = 0
 415                         else:
 416                                 existing_size = mystat.st_size
 417                         remaining_size = int(checksums[myfile]["size"]) - existing_size
 418                         if remaining_size > 0:
 419                                 # Assume the download is resumable.
 420                                 filesdict[myfile] = remaining_size
 421                         elif remaining_size < 0:
 422                                 # The existing file is too large and therefore corrupt.
 423                                 filesdict[myfile] = int(checksums[myfile]["size"])
 424                 return filesdict
 425
 426         def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False):
 427                 if not useflags:
 428                         if mysettings:
 429                                 useflags = mysettings["USE"].split()
 430                 myuri, myfiles = self.getfetchlist(mypkg, useflags=useflags, mysettings=mysettings, all=all)
 431                 myebuild = self.findname(mypkg)
 432                 pkgdir = os.path.dirname(myebuild)
 433                 mf = Manifest(pkgdir, self.mysettings["DISTDIR"])
 434                 mysums = mf.getDigests()
 435
 436                 failures = {}
 437                 for x in myfiles:
 438                         if not mysums or x not in mysums:
 439                                 ok     = False
 440                                 reason = "digest missing"
 441                         else:
 442                                 try:
 443                                         ok, reason = portage.checksum.verify_all(
 444                                                 os.path.join(self.mysettings["DISTDIR"], x), mysums[x])
 445                                 except FileNotFound, e:
 446                                         ok = False
 447                                         reason = "File Not Found: '%s'" % str(e)
 448                         if not ok:
 449                                 failures[x] = reason
 450                 if failures:
 451                         return False
 452                 return True
 453
 454         def cpv_exists(self, mykey):
 455                 "Tells us whether an actual ebuild exists on disk (no masking)"
 456                 cps2 = mykey.split("/")
 457                 cps = catpkgsplit(mykey, silent=0)
 458                 if not cps:
 459                         #invalid cat/pkg-v
 460                         return 0
 461                 if self.findname(cps[0] + "/" + cps2[1]):
 462                         return 1
 463                 else:
 464                         return 0
 465
 466         def cp_all(self):
 467                 "returns a list of all keys in our tree"
 468                 d = {}
 469                 for x in self.mysettings.categories:
 470                         for oroot in self.porttrees:
 471                                 for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1):
 472                                         d[x+"/"+y] = None
 473                 l = d.keys()
 474                 l.sort()
 475                 return l
 476
 477         def p_list(self,mycp):
 478                 d={}
 479                 for oroot in self.porttrees:
 480                         for x in listdir(oroot+"/"+mycp,EmptyOnError=1,ignorecvs=1):
 481                                 if x[-7:]==".ebuild":
 482                                         d[x[:-7]] = None
 483                 return d.keys()
 484
 485         def cp_list(self, mycp, use_cache=1, mytree=None):
 486                 mysplit = mycp.split("/")
 487                 invalid_category = mysplit[0] not in self._categories
 488                 d={}
 489                 if mytree:
 490                         mytrees = [mytree]
 491                 else:
 492                         mytrees = self.porttrees
 493                 for oroot in mytrees:
 494                         for x in listdir(oroot+"/"+mycp, EmptyOnError=1, ignorecvs=1):
 495                                 if x.endswith(".ebuild"):
 496                                         pf = x[:-7]
 497                                         ps = pkgsplit(pf)
 498                                         if not ps:
 499                                                 writemsg("\nInvalid ebuild name: %s\n" % \
 500                                                         os.path.join(oroot, mycp, x), noiselevel=-1)
 501                                                 continue
 502                                         d[mysplit[0]+"/"+pf] = None
 503                 if invalid_category and d:
 504                         writemsg(("\n!!! '%s' has a category that is not listed in " + \
 505                                 "/etc/portage/categories\n") % mycp, noiselevel=-1)
 506                         return []
 507                 return d.keys()
 508
 509         def freeze(self):
 510                 for x in ["list-visible", "bestmatch-visible", "match-visible", "match-all"]:
 511                         self.xcache[x]={}
 512                 self.frozen=1
 513
 514         def melt(self):
 515                 self.xcache = {}
 516                 self.frozen = 0
 517
 518         def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
 519                 "caching match function; very trick stuff"
 520                 #if no updates are being made to the tree, we can consult our xcache...
 521                 if self.frozen:
 522                         try:
 523                                 return self.xcache[level][origdep][:]
 524                         except KeyError:
 525                                 pass
 526
 527                 if not mydep:
 528                         #this stuff only runs on first call of xmatch()
 529                         #create mydep, mykey from origdep
 530                         mydep = dep_expand(origdep, mydb=self, settings=self.mysettings)
 531                         mykey = dep_getkey(mydep)
 532
 533                 if level == "list-visible":
 534                         #a list of all visible packages, not called directly (just by xmatch())
 535                         #myval = self.visible(self.cp_list(mykey))
 536
 537                         myval = self.gvisible(self.visible(self.cp_list(mykey)))
 538                 elif level == "bestmatch-visible":
 539                         #dep match -- best match of all visible packages
 540                         #get all visible matches (from xmatch()), then choose the best one
 541
 542                         myval = best(self.xmatch("match-visible", None, mydep=mydep, mykey=mykey))
 543                 elif level == "bestmatch-list":
 544                         #dep match -- find best match but restrict search to sublist
 545                         #no point in calling xmatch again since we're not caching list deps
 546
 547                         myval = best(match_from_list(mydep, mylist))
 548                 elif level == "match-list":
 549                         #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible())
 550
 551                         myval = match_from_list(mydep, mylist)
 552                 elif level == "match-visible":
 553                         #dep match -- find all visible matches
 554                         #get all visible packages, then get the matching ones
 555
 556                         myval = match_from_list(mydep,
 557                                 self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey))
 558                 elif level == "match-all":
 559                         #match *all* visible *and* masked packages
 560
 561                         myval = match_from_list(mydep, self.cp_list(mykey))
 562                 else:
 563                         print "ERROR: xmatch doesn't handle", level, "query!"
 564                         raise KeyError
 565                 myslot = dep_getslot(mydep)
 566                 if myslot is not None:
 567                         slotmatches = []
 568                         for cpv in myval:
 569                                 try:
 570                                         if self.aux_get(cpv, ["SLOT"])[0] == myslot:
 571                                                 slotmatches.append(cpv)
 572                                 except KeyError:
 573                                         pass # ebuild masked by corruption
 574                         myval = slotmatches
 575                 if self.frozen and (level not in ["match-list", "bestmatch-list"]):
 576                         self.xcache[level][mydep] = myval
 577                         if origdep and origdep != mydep:
 578                                 self.xcache[level][origdep] = myval
 579                 return myval[:]
 580
 581         def match(self, mydep, use_cache=1):
 582                 return self.xmatch("match-visible", mydep)
 583
 584         def visible(self, mylist):
 585                 """two functions in one.  Accepts a list of cpv values and uses the package.mask *and*
 586                 packages file to remove invisible entries, returning remaining items.  This function assumes
 587                 that all entries in mylist have the same category and package name."""
 588                 if (mylist is None) or (len(mylist) == 0):
 589                         return []
 590                 newlist = mylist[:]
 591                 #first, we mask out packages in the package.mask file
 592                 mykey = newlist[0]
 593                 cpv = catpkgsplit(mykey)
 594                 if not cpv:
 595                         #invalid cat/pkg-v
 596                         print "visible(): invalid cat/pkg-v:", mykey
 597                         return []
 598                 mycp = cpv[0] + "/" + cpv[1]
 599                 maskdict = self.mysettings.pmaskdict
 600                 unmaskdict = self.mysettings.punmaskdict
 601                 if maskdict.has_key(mycp):
 602                         for x in maskdict[mycp]:
 603                                 mymatches = self.xmatch("match-all", x)
 604                                 if mymatches is None:
 605                                         #error in package.mask file; print warning and continue:
 606                                         print "visible(): package.mask entry \"" + x + "\" is invalid, ignoring..."
 607                                         continue
 608                                 for y in mymatches:
 609                                         unmask = 0
 610                                         if unmaskdict.has_key(mycp):
 611                                                 for z in unmaskdict[mycp]:
 612                                                         mymatches_unmask = self.xmatch("match-all",z)
 613                                                         if y in mymatches_unmask:
 614                                                                 unmask = 1
 615                                                                 break
 616                                         if unmask == 0:
 617                                                 try:
 618                                                         newlist.remove(y)
 619                                                 except ValueError:
 620                                                         pass
 621
 622                 profile_atoms = self.mysettings.prevmaskdict.get(mycp)
 623                 if profile_atoms:
 624                         for x in profile_atoms:
 625                                 #important: only match against the still-unmasked entries...
 626                                 #notice how we pass "newlist" to the xmatch() call below....
 627                                 #Without this, ~ deps in the packages files are broken.
 628                                 mymatches=self.xmatch("match-list",x,mylist=newlist)
 629                                 if mymatches is None:
 630                                         #error in packages file; print warning and continue:
 631                                         print "emerge: visible(): profile packages entry \""+x+"\" is invalid, ignoring..."
 632                                         continue
 633                                 newlist = [cpv for cpv in newlist if cpv in mymatches]
 634
 635                 return newlist
 636
 637         def gvisible(self,mylist):
 638                 "strip out group-masked (not in current group) entries"
 639
 640                 if mylist is None:
 641                         return []
 642                 newlist=[]
 643
 644                 accept_keywords = self.mysettings["ACCEPT_KEYWORDS"].split()
 645                 pkgdict = self.mysettings.pkeywordsdict
 646                 aux_keys = ["KEYWORDS", "LICENSE", "EAPI", "SLOT"]
 647
 648                 # Hack: Need to check the env directly here as otherwise stacking
 649                 # doesn't work properly as negative values are lost in the config
 650                 # object (bug #139600)
 651                 egroups = self.mysettings.configdict["backupenv"].get(
 652                         "ACCEPT_KEYWORDS", "").split()
 653
 654                 for mycpv in mylist:
 655                         try:
 656                                 keys, licenses, eapi, slot = self.aux_get(mycpv, aux_keys)
 657                         except KeyError:
 658                                 continue
 659                         except PortageException, e:
 660                                 writemsg("!!! Error: aux_get('%s', %s)\n" % (mycpv, aux_keys),
 661                                         mycpv, noiselevel=-1)
 662                                 writemsg("!!! %s\n" % str(e), noiselevel=-1)
 663                                 del e
 664                                 continue
 665                         mygroups = keys.split()
 666                         # Repoman may modify this attribute as necessary.
 667                         pgroups = accept_keywords[:]
 668                         match=0
 669                         cp = dep_getkey(mycpv)
 670                         if pkgdict.has_key(cp):
 671                                 cpv_slot = "%s:%s" % (mycpv, slot)
 672                                 matches = match_to_list(cpv_slot, pkgdict[cp].keys())
 673                                 for atom in matches:
 674                                         pgroups.extend(pkgdict[cp][atom])
 675                                 pgroups.extend(egroups)
 676                                 if matches:
 677                                         # normalize pgroups with incrementals logic so it
 678                                         # matches ACCEPT_KEYWORDS behavior
 679                                         inc_pgroups = []
 680                                         for x in pgroups:
 681                                                 if x == "-*":
 682                                                         inc_pgroups = []
 683                                                 elif x[0] == "-":
 684                                                         try:
 685                                                                 inc_pgroups.remove(x[1:])
 686                                                         except ValueError:
 687                                                                 pass
 688                                                 elif x not in inc_pgroups:
 689                                                         inc_pgroups.append(x)
 690                                         pgroups = inc_pgroups
 691                                         del inc_pgroups
 692                         hasstable = False
 693                         hastesting = False
 694                         for gp in mygroups:
 695                                 if gp == "*" or (gp == "-*" and len(mygroups) == 1):
 696                                         writemsg("--- WARNING: Package '%s' uses '%s' keyword.\n" % (mycpv, gp),
 697                                                 noiselevel=-1)
 698                                         if gp == "*":
 699                                                 match = 1
 700                                                 break
 701                                 elif gp in pgroups:
 702                                         match=1
 703                                         break
 704                                 elif gp[0] == "~":
 705                                         hastesting = True
 706                                 elif gp[0] != "-":
 707                                         hasstable = True
 708                         if not match and ((hastesting and "~*" in pgroups) or (hasstable and "*" in pgroups) or "**" in pgroups):
 709                                 match=1
 710                         uselist = []
 711                         if "?" in licenses:
 712                                 self.doebuild_settings.setcpv(mycpv, mydb=self)
 713                                 uselist = self.doebuild_settings.get("USE", "").split()
 714                         try:
 715                                 if self.mysettings.getMissingLicenses(
 716                                         licenses, mycpv, uselist):
 717                                         match = 0
 718                         except InvalidDependString:
 719                                 match = 0
 720                         if match and eapi_is_supported(eapi):
 721                                 newlist.append(mycpv)
 722                 return newlist
 723
 724
 725 def close_portdbapi_caches():
 726         for i in portdbapi.portdbapi_instances:
 727                 i.close_caches()
 728
 729
 730 class portagetree(object):
 731         def __init__(self, root="/", virtual=None, clone=None, settings=None):
 732                 """
 733                 Constructor for a PortageTree
 734
 735                 @param root: ${ROOT}, defaults to '/', see make.conf(5)
 736                 @type root: String/Path
 737                 @param virtual: UNUSED
 738                 @type virtual: No Idea
 739                 @param clone: Set this if you want a copy of Clone
 740                 @type clone: Existing portagetree Instance
 741                 @param settings: Portage Configuration object (portage.settings)
 742                 @type settings: Instance of portage.config
 743                 """
 744
 745                 if clone:
 746                         writemsg("portagetree.__init__(): deprecated " + \
 747                                 "use of clone parameter\n", noiselevel=-1)
 748                         self.root = clone.root
 749                         self.portroot = clone.portroot
 750                         self.pkglines = clone.pkglines
 751                 else:
 752                         self.root = root
 753                         if settings is None:
 754                                 from portage import settings
 755                         self.settings = settings
 756                         self.portroot = settings["PORTDIR"]
 757                         self.virtual = virtual
 758                         self.dbapi = portdbapi(
 759                                 settings["PORTDIR"], mysettings=settings)
 760
 761         def dep_bestmatch(self,mydep):
 762                 "compatibility method"
 763                 mymatch = self.dbapi.xmatch("bestmatch-visible",mydep)
 764                 if mymatch is None:
 765                         return ""
 766                 return mymatch
 767
 768         def dep_match(self,mydep):
 769                 "compatibility method"
 770                 mymatch = self.dbapi.xmatch("match-visible",mydep)
 771                 if mymatch is None:
 772                         return []
 773                 return mymatch
 774
 775         def exists_specific(self,cpv):
 776                 return self.dbapi.cpv_exists(cpv)
 777
 778         def getallnodes(self):
 779                 """new behavior: these are all *unmasked* nodes.  There may or may not be available
 780                 masked package for nodes in this nodes list."""
 781                 return self.dbapi.cp_all()
 782
 783         def getname(self, pkgname):
 784                 "returns file location for this particular package (DEPRECATED)"
 785                 if not pkgname:
 786                         return ""
 787                 mysplit = pkgname.split("/")
 788                 psplit = pkgsplit(mysplit[1])
 789                 return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild"
 790
 791         def resolve_specific(self, myspec):
 792                 cps = catpkgsplit(myspec)
 793                 if not cps:
 794                         return None
 795                 mykey = key_expand(cps[0]+"/"+cps[1], mydb=self.dbapi,
 796                         settings=self.settings)
 797                 mykey = mykey + "-" + cps[2]
 798                 if cps[3] != "r0":
 799                         mykey = mykey + "-" + cps[3]
 800                 return mykey
 801
 802         def depcheck(self, mycheck, use="yes", myusesplit=None):
 803                 return dep_check(mycheck, self.dbapi, use=use, myuse=myusesplit)
 804
 805         def getslot(self,mycatpkg):
 806                 "Get a slot for a catpkg; assume it exists."
 807                 myslot = ""
 808                 try:
 809                         myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0]
 810                 except SystemExit, e:
 811                         raise
 812                 except Exception, e:
 813                         pass
 814                 return myslot
 815