pym/portage/dbapi/porttree.py

   1 from portage.cache.cache_errors import CacheError
   2 from portage.const import REPO_NAME_LOC
   3 from portage.data import portage_gid, secpass
   4 from portage.dbapi import dbapi
   5 from portage.dep import use_reduce, paren_reduce, dep_getslot, dep_getkey, \
   6         match_from_list, match_to_list
   7 from portage.exception import OperationNotPermitted, PortageException, \
   8         UntrustedSignature, SecurityViolation, InvalidSignature, MissingSignature, \
   9         FileNotFound
  10 from portage.manifest import Manifest
  11 from portage.output import red
  12 from portage.util import ensure_dirs, writemsg, apply_recursive_permissions
  13 from portage.versions import pkgsplit, catpkgsplit, best
  14
  15 import portage.gpg, portage.checksum
  16
  17 from portage import eclass_cache, auxdbkeys, auxdbkeylen, doebuild, flatten, \
  18         listdir, dep_expand, eapi_is_supported, key_expand, dep_check, config
  19
  20 import os, stat, sys
  21
  22
  23 class portdbapi(dbapi):
  24         """this tree will scan a portage directory located at root (passed to init)"""
  25         portdbapi_instances = []
  26
  27         def __init__(self, porttree_root, mysettings=None):
  28                 portdbapi.portdbapi_instances.append(self)
  29
  30                 if mysettings:
  31                         self.mysettings = mysettings
  32                 else:
  33                         from portage import settings
  34                         self.mysettings = config(clone=settings)
  35                 self._categories = set(self.mysettings.categories)
  36                 # This is strictly for use in aux_get() doebuild calls when metadata
  37                 # is generated by the depend phase.  It's safest to use a clone for
  38                 # this purpose because doebuild makes many changes to the config
  39                 # instance that is passed in.
  40                 self.doebuild_settings = config(clone=self.mysettings)
  41
  42                 self.manifestVerifyLevel = None
  43                 self.manifestVerifier = None
  44                 self.manifestCache = {}    # {location: [stat, md5]}
  45                 self.manifestMissingCache = []
  46
  47                 if "gpg" in self.mysettings.features:
  48                         self.manifestVerifyLevel = portage.gpg.EXISTS
  49                         if "strict" in self.mysettings.features:
  50                                 self.manifestVerifyLevel = portage.gpg.MARGINAL
  51                                 self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel)
  52                         elif "severe" in self.mysettings.features:
  53                                 self.manifestVerifyLevel = portage.gpg.TRUSTED
  54                                 self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", requireSignedRing=True, minimumTrust=self.manifestVerifyLevel)
  55                         else:
  56                                 self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel)
  57
  58                 #self.root=settings["PORTDIR"]
  59                 self.porttree_root = os.path.realpath(porttree_root)
  60
  61                 self.depcachedir = self.mysettings.depcachedir[:]
  62
  63                 self.tmpfs = self.mysettings["PORTAGE_TMPFS"]
  64                 if self.tmpfs and not os.path.exists(self.tmpfs):
  65                         self.tmpfs = None
  66                 if self.tmpfs and not os.access(self.tmpfs, os.W_OK):
  67                         self.tmpfs = None
  68                 if self.tmpfs and not os.access(self.tmpfs, os.R_OK):
  69                         self.tmpfs = None
  70
  71                 self.eclassdb = eclass_cache.cache(self.porttree_root,
  72                         overlays=self.mysettings["PORTDIR_OVERLAY"].split())
  73
  74                 self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule")
  75
  76                 #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
  77                 self.xcache = {}
  78                 self.frozen = 0
  79
  80                 self.porttrees = [self.porttree_root] + \
  81                         [os.path.realpath(t) for t in self.mysettings["PORTDIR_OVERLAY"].split()]
  82                 self.treemap = {}
  83                 for path in self.porttrees:
  84                         repo_name_path = os.path.join(path, REPO_NAME_LOC)
  85                         try:
  86                                 repo_name = open(repo_name_path, 'r').readline().strip()
  87                                 self.treemap[repo_name] = path
  88                         except (OSError,IOError):
  89                                 pass
  90
  91                 self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule")
  92                 self.auxdb = {}
  93                 self._init_cache_dirs()
  94                 # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
  95                 # ~harring
  96                 filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys)
  97                 if secpass < 1:
  98                         from portage.cache import metadata_overlay, volatile
  99                         for x in self.porttrees:
 100                                 db_ro = self.auxdbmodule(self.depcachedir, x,
 101                                         filtered_auxdbkeys, gid=portage_gid, readonly=True)
 102                                 self.auxdb[x] = metadata_overlay.database(
 103                                         self.depcachedir, x, filtered_auxdbkeys,
 104                                         gid=portage_gid, db_rw=volatile.database,
 105                                         db_ro=db_ro)
 106                 else:
 107                         for x in self.porttrees:
 108                                 # location, label, auxdbkeys
 109                                 self.auxdb[x] = self.auxdbmodule(
 110                                         self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid)
 111                 # Selectively cache metadata in order to optimize dep matching.
 112                 self._aux_cache_keys = set(["EAPI", "KEYWORDS", "SLOT"])
 113                 self._aux_cache = {}
 114
 115         def _init_cache_dirs(self):
 116                 """Create /var/cache/edb/dep and adjust permissions for the portage
 117                 group."""
 118
 119                 dirmode = 02070
 120                 filemode =   060
 121                 modemask =    02
 122
 123                 try:
 124                         for mydir in (self.depcachedir,):
 125                                 if ensure_dirs(mydir, gid=portage_gid, mode=dirmode, mask=modemask):
 126                                         writemsg("Adjusting permissions recursively: '%s'\n" % mydir,
 127                                                 noiselevel=-1)
 128                                         def onerror(e):
 129                                                 raise # bail out on the first error that occurs during recursion
 130                                         if not apply_recursive_permissions(mydir,
 131                                                 gid=portage_gid, dirmode=dirmode, dirmask=modemask,
 132                                                 filemode=filemode, filemask=modemask, onerror=onerror):
 133                                                 raise OperationNotPermitted(
 134                                                         "Failed to apply recursive permissions for the portage group.")
 135                 except PortageException, e:
 136                         pass
 137
 138         def close_caches(self):
 139                 for x in self.auxdb.keys():
 140                         self.auxdb[x].sync()
 141                 self.auxdb.clear()
 142
 143         def flush_cache(self):
 144                 for x in self.auxdb.values():
 145                         x.sync()
 146
 147         def finddigest(self,mycpv):
 148                 try:
 149                         mydig = self.findname2(mycpv)[0]
 150                         if not mydig:
 151                                 return ""
 152                         mydigs = mydig.split("/")[:-1]
 153                         mydig = "/".join(mydigs)
 154                         mysplit = mycpv.split("/")
 155                 except OSError:
 156                         return ""
 157                 return mydig+"/files/digest-"+mysplit[-1]
 158
 159         def findname(self,mycpv):
 160                 return self.findname2(mycpv)[0]
 161
 162         def getRepositoryPath(self, repository_id):
 163                 """
 164                 This function is required for GLEP 42 compliance; given a valid repository ID
 165                 it must return a path to the repository
 166                 TreeMap = { id:path }
 167                 """
 168                 if repository_id in self.treemap:
 169                         return self.treemap[repository_id]
 170                 return None
 171
 172         def getRepositories(self):
 173                 """
 174                 This function is required for GLEP 42 compliance; it will return a list of
 175                 repository ID's
 176                 TreeMap = {id: path}
 177                 """
 178                 return [k for k in self.treemap.keys() if k]
 179
 180         def findname2(self, mycpv, mytree=None):
 181                 """
 182                 Returns the location of the CPV, and what overlay it was in.
 183                 Searches overlays first, then PORTDIR; this allows us to return the first
 184                 matching file.  As opposed to starting in portdir and then doing overlays
 185                 second, we would have to exhaustively search the overlays until we found
 186                 the file we wanted.
 187                 """
 188                 if not mycpv:
 189                         return "",0
 190                 mysplit = mycpv.split("/")
 191                 psplit = pkgsplit(mysplit[1])
 192
 193                 if mytree:
 194                         mytrees = [mytree]
 195                 else:
 196                         mytrees = self.porttrees[:]
 197                         mytrees.reverse()
 198                 if psplit:
 199                         for x in mytrees:
 200                                 file=x+"/"+mysplit[0]+"/"+psplit[0]+"/"+mysplit[1]+".ebuild"
 201                                 if os.access(file, os.R_OK):
 202                                         return[file, x]
 203                 return None, 0
 204
 205         def aux_get(self, mycpv, mylist, mytree=None):
 206                 "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
 207                 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
 208                 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
 209                 cache_me = False
 210                 if not mytree and not set(mylist).difference(self._aux_cache_keys):
 211                         aux_cache = self._aux_cache.get(mycpv)
 212                         if aux_cache is not None:
 213                                 return [aux_cache[x] for x in mylist]
 214                         cache_me = True
 215                 global auxdbkeys, auxdbkeylen
 216                 cat,pkg = mycpv.split("/", 1)
 217
 218                 myebuild, mylocation = self.findname2(mycpv, mytree)
 219
 220                 if not myebuild:
 221                         writemsg("!!! aux_get(): ebuild path for '%(cpv)s' not specified:\n" % {"cpv":mycpv},
 222                                 noiselevel=1)
 223                         writemsg("!!!            %s\n" % myebuild, noiselevel=1)
 224                         raise KeyError(mycpv)
 225
 226                 myManifestPath = "/".join(myebuild.split("/")[:-1])+"/Manifest"
 227                 if "gpg" in self.mysettings.features:
 228                         try:
 229                                 mys = portage.gpg.fileStats(myManifestPath)
 230                                 if (myManifestPath in self.manifestCache) and \
 231                                    (self.manifestCache[myManifestPath] == mys):
 232                                         pass
 233                                 elif self.manifestVerifier:
 234                                         if not self.manifestVerifier.verify(myManifestPath):
 235                                                 # Verification failed the desired level.
 236                                                 raise UntrustedSignature, "Untrusted Manifest: %(manifest)s" % {"manifest":myManifestPath}
 237
 238                                 if ("severe" in self.mysettings.features) and \
 239                                    (mys != portage.gpg.fileStats(myManifestPath)):
 240                                         raise SecurityViolation, "Manifest changed: %(manifest)s" % {"manifest":myManifestPath}
 241
 242                         except InvalidSignature, e:
 243                                 if ("strict" in self.mysettings.features) or \
 244                                    ("severe" in self.mysettings.features):
 245                                         raise
 246                                 writemsg("!!! INVALID MANIFEST SIGNATURE DETECTED: %(manifest)s\n" % {"manifest":myManifestPath})
 247                         except MissingSignature, e:
 248                                 if ("severe" in self.mysettings.features):
 249                                         raise
 250                                 if ("strict" in self.mysettings.features):
 251                                         if myManifestPath not in self.manifestMissingCache:
 252                                                 writemsg("!!! WARNING: Missing signature in: %(manifest)s\n" % {"manifest":myManifestPath})
 253                                                 self.manifestMissingCache.insert(0,myManifestPath)
 254                         except (OSError, FileNotFound), e:
 255                                 if ("strict" in self.mysettings.features) or \
 256                                    ("severe" in self.mysettings.features):
 257                                         raise SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)}
 258                                 writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath},
 259                                         noiselevel=-1)
 260
 261
 262                 if os.access(myebuild, os.R_OK):
 263                         emtime = os.stat(myebuild)[stat.ST_MTIME]
 264                 else:
 265                         writemsg("!!! aux_get(): ebuild for '%(cpv)s' does not exist at:\n" % {"cpv":mycpv},
 266                                 noiselevel=-1)
 267                         writemsg("!!!            %s\n" % myebuild,
 268                                 noiselevel=-1)
 269                         raise KeyError
 270
 271                 try:
 272                         mydata = self.auxdb[mylocation][mycpv]
 273                         if emtime != long(mydata.get("_mtime_", 0)):
 274                                 doregen = True
 275                         elif len(mydata.get("_eclasses_", [])) > 0:
 276                                 doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"])
 277                         else:
 278                                 doregen = False
 279
 280                 except KeyError:
 281                         doregen = True
 282                 except CacheError:
 283                         doregen = True
 284                         try:
 285                                 del self.auxdb[mylocation][mycpv]
 286                         except KeyError:
 287                                 pass
 288
 289                 writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2)
 290
 291                 if doregen:
 292                         writemsg("doregen: %s %s\n" % (doregen, mycpv), 2)
 293                         writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n", 1)
 294
 295                         self.doebuild_settings.reset()
 296                         mydata = {}
 297                         myret = doebuild(myebuild, "depend",
 298                                 self.doebuild_settings["ROOT"], self.doebuild_settings,
 299                                 dbkey=mydata, tree="porttree", mydbapi=self)
 300                         if myret != os.EX_OK:
 301                                 raise KeyError(mycpv)
 302
 303                         if "EAPI" not in mydata or not mydata["EAPI"].strip():
 304                                 mydata["EAPI"] = "0"
 305
 306                         if not eapi_is_supported(mydata["EAPI"]):
 307                                 # if newer version, wipe everything and negate eapi
 308                                 eapi = mydata["EAPI"]
 309                                 mydata = {}
 310                                 map(lambda x: mydata.setdefault(x, ""), auxdbkeys)
 311                                 mydata["EAPI"] = "-"+eapi
 312
 313                         if mydata.get("INHERITED", False):
 314                                 mydata["_eclasses_"] = self.eclassdb.get_eclass_data(mydata["INHERITED"].split())
 315                         else:
 316                                 mydata["_eclasses_"] = {}
 317
 318                         del mydata["INHERITED"]
 319
 320                         mydata["_mtime_"] = emtime
 321
 322                         self.auxdb[mylocation][mycpv] = mydata
 323
 324                 if not mydata.setdefault("EAPI", "0"):
 325                         mydata["EAPI"] = "0"
 326
 327                 #finally, we look at our internal cache entry and return the requested data.
 328                 returnme = []
 329                 for x in mylist:
 330                         if x == "INHERITED":
 331                                 returnme.append(' '.join(mydata.get("_eclasses_", {}).keys()))
 332                         else:
 333                                 returnme.append(mydata.get(x,""))
 334
 335                 if cache_me:
 336                         aux_cache = {}
 337                         for x in self._aux_cache_keys:
 338                                 aux_cache[x] = mydata.get(x, "")
 339                         self._aux_cache[mycpv] = aux_cache
 340
 341                 return returnme
 342
 343         def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None):
 344                 if mysettings is None:
 345                         mysettings = self.mysettings
 346                 try:
 347                         myuris = self.aux_get(mypkg, ["SRC_URI"], mytree=mytree)[0]
 348                 except KeyError:
 349                         print red("getfetchlist():")+" aux_get() error reading "+mypkg+"; aborting."
 350                         sys.exit(1)
 351
 352                 if useflags is None:
 353                         useflags = mysettings["USE"].split()
 354
 355                 myurilist = paren_reduce(myuris)
 356                 myurilist = use_reduce(myurilist, uselist=useflags, matchall=all)
 357                 newuris = flatten(myurilist)
 358
 359                 myfiles = []
 360                 for x in newuris:
 361                         mya = os.path.basename(x)
 362                         if not mya in myfiles:
 363                                 myfiles.append(mya)
 364                 return [newuris, myfiles]
 365
 366         def getfetchsizes(self, mypkg, useflags=None, debug=0):
 367                 # returns a filename:size dictionnary of remaining downloads
 368                 myebuild = self.findname(mypkg)
 369                 pkgdir = os.path.dirname(myebuild)
 370                 mf = Manifest(pkgdir, self.mysettings["DISTDIR"])
 371                 checksums = mf.getDigests()
 372                 if not checksums:
 373                         if debug:
 374                                 print "[empty/missing/bad digest]: "+mypkg
 375                         return None
 376                 filesdict={}
 377                 if useflags is None:
 378                         myuris, myfiles = self.getfetchlist(mypkg,all=1)
 379                 else:
 380                         myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags)
 381                 #XXX: maybe this should be improved: take partial downloads
 382                 # into account? check checksums?
 383                 for myfile in myfiles:
 384                         if myfile not in checksums:
 385                                 if debug:
 386                                         writemsg("[bad digest]: missing %s for %s\n" % (myfile, mypkg))
 387                                 continue
 388                         file_path = os.path.join(self.mysettings["DISTDIR"], myfile)
 389                         mystat = None
 390                         try:
 391                                 mystat = os.stat(file_path)
 392                         except OSError, e:
 393                                 pass
 394                         if mystat is None:
 395                                 existing_size = 0
 396                         else:
 397                                 existing_size = mystat.st_size
 398                         remaining_size = int(checksums[myfile]["size"]) - existing_size
 399                         if remaining_size > 0:
 400                                 # Assume the download is resumable.
 401                                 filesdict[myfile] = remaining_size
 402                         elif remaining_size < 0:
 403                                 # The existing file is too large and therefore corrupt.
 404                                 filesdict[myfile] = int(checksums[myfile]["size"])
 405                 return filesdict
 406
 407         def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False):
 408                 if not useflags:
 409                         if mysettings:
 410                                 useflags = mysettings["USE"].split()
 411                 myuri, myfiles = self.getfetchlist(mypkg, useflags=useflags, mysettings=mysettings, all=all)
 412                 myebuild = self.findname(mypkg)
 413                 pkgdir = os.path.dirname(myebuild)
 414                 mf = Manifest(pkgdir, self.mysettings["DISTDIR"])
 415                 mysums = mf.getDigests()
 416
 417                 failures = {}
 418                 for x in myfiles:
 419                         if not mysums or x not in mysums:
 420                                 ok     = False
 421                                 reason = "digest missing"
 422                         else:
 423                                 try:
 424                                         ok, reason = portage.checksum.verify_all(
 425                                                 os.path.join(self.mysettings["DISTDIR"], x), mysums[x])
 426                                 except FileNotFound, e:
 427                                         ok = False
 428                                         reason = "File Not Found: '%s'" % str(e)
 429                         if not ok:
 430                                 failures[x] = reason
 431                 if failures:
 432                         return False
 433                 return True
 434
 435         def getsize(self, mypkg, useflags=None, debug=0):
 436                 # returns the total size of remaining downloads
 437                 #
 438                 # we use getfetchsizes() now, so this function would be obsoleted
 439                 #
 440                 filesdict = self.getfetchsizes(mypkg, useflags=useflags, debug=debug)
 441                 if filesdict is None:
 442                         return "[empty/missing/bad digest]"
 443                 mysize=0
 444                 for myfile in filesdict.keys():
 445                         mysum += filesdict[myfile]
 446                 return mysum
 447
 448         def cpv_exists(self, mykey):
 449                 "Tells us whether an actual ebuild exists on disk (no masking)"
 450                 cps2 = mykey.split("/")
 451                 cps = catpkgsplit(mykey, silent=0)
 452                 if not cps:
 453                         #invalid cat/pkg-v
 454                         return 0
 455                 if self.findname(cps[0] + "/" + cps2[1]):
 456                         return 1
 457                 else:
 458                         return 0
 459
 460         def cp_all(self):
 461                 "returns a list of all keys in our tree"
 462                 d = {}
 463                 for x in self.mysettings.categories:
 464                         for oroot in self.porttrees:
 465                                 for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1):
 466                                         d[x+"/"+y] = None
 467                 l = d.keys()
 468                 l.sort()
 469                 return l
 470
 471         def p_list(self,mycp):
 472                 d={}
 473                 for oroot in self.porttrees:
 474                         for x in listdir(oroot+"/"+mycp,EmptyOnError=1,ignorecvs=1):
 475                                 if x[-7:]==".ebuild":
 476                                         d[x[:-7]] = None
 477                 return d.keys()
 478
 479         def cp_list(self, mycp, use_cache=1, mytree=None):
 480                 mysplit = mycp.split("/")
 481                 invalid_category = mysplit[0] not in self._categories
 482                 d={}
 483                 if mytree:
 484                         mytrees = [mytree]
 485                 else:
 486                         mytrees = self.porttrees
 487                 for oroot in mytrees:
 488                         for x in listdir(oroot+"/"+mycp, EmptyOnError=1, ignorecvs=1):
 489                                 if x.endswith(".ebuild"):
 490                                         pf = x[:-7]
 491                                         ps = pkgsplit(pf)
 492                                         if not ps:
 493                                                 writemsg("\nInvalid ebuild name: %s\n" % \
 494                                                         os.path.join(oroot, mycp, x), noiselevel=-1)
 495                                                 continue
 496                                         d[mysplit[0]+"/"+pf] = None
 497                 if invalid_category and d:
 498                         writemsg(("\n!!! '%s' has a category that is not listed in " + \
 499                                 "/etc/portage/categories\n") % mycp, noiselevel=-1)
 500                         return []
 501                 return d.keys()
 502
 503         def freeze(self):
 504                 for x in ["list-visible", "bestmatch-visible", "match-visible", "match-all"]:
 505                         self.xcache[x]={}
 506                 self.frozen=1
 507
 508         def melt(self):
 509                 self.xcache = {}
 510                 self.frozen = 0
 511
 512         def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
 513                 "caching match function; very trick stuff"
 514                 #if no updates are being made to the tree, we can consult our xcache...
 515                 if self.frozen:
 516                         try:
 517                                 return self.xcache[level][origdep][:]
 518                         except KeyError:
 519                                 pass
 520
 521                 if not mydep:
 522                         #this stuff only runs on first call of xmatch()
 523                         #create mydep, mykey from origdep
 524                         mydep = dep_expand(origdep, mydb=self, settings=self.mysettings)
 525                         mykey = dep_getkey(mydep)
 526
 527                 if level == "list-visible":
 528                         #a list of all visible packages, not called directly (just by xmatch())
 529                         #myval = self.visible(self.cp_list(mykey))
 530
 531                         myval = self.gvisible(self.visible(self.cp_list(mykey)))
 532                 elif level == "bestmatch-visible":
 533                         #dep match -- best match of all visible packages
 534                         #get all visible matches (from xmatch()), then choose the best one
 535
 536                         myval = best(self.xmatch("match-visible", None, mydep=mydep, mykey=mykey))
 537                 elif level == "bestmatch-list":
 538                         #dep match -- find best match but restrict search to sublist
 539                         #no point in calling xmatch again since we're not caching list deps
 540
 541                         myval = best(match_from_list(mydep, mylist))
 542                 elif level == "match-list":
 543                         #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible())
 544
 545                         myval = match_from_list(mydep, mylist)
 546                 elif level == "match-visible":
 547                         #dep match -- find all visible matches
 548                         #get all visible packages, then get the matching ones
 549
 550                         myval = match_from_list(mydep,
 551                                 self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey))
 552                 elif level == "match-all":
 553                         #match *all* visible *and* masked packages
 554
 555                         myval = match_from_list(mydep, self.cp_list(mykey))
 556                 else:
 557                         print "ERROR: xmatch doesn't handle", level, "query!"
 558                         raise KeyError
 559                 myslot = dep_getslot(mydep)
 560                 if myslot is not None:
 561                         slotmatches = []
 562                         for cpv in myval:
 563                                 try:
 564                                         if self.aux_get(cpv, ["SLOT"])[0] == myslot:
 565                                                 slotmatches.append(cpv)
 566                                 except KeyError:
 567                                         pass # ebuild masked by corruption
 568                         myval = slotmatches
 569                 if self.frozen and (level not in ["match-list", "bestmatch-list"]):
 570                         self.xcache[level][mydep] = myval
 571                         if origdep and origdep != mydep:
 572                                 self.xcache[level][origdep] = myval
 573                 return myval[:]
 574
 575         def match(self, mydep, use_cache=1):
 576                 return self.xmatch("match-visible", mydep)
 577
 578         def visible(self, mylist):
 579                 """two functions in one.  Accepts a list of cpv values and uses the package.mask *and*
 580                 packages file to remove invisible entries, returning remaining items.  This function assumes
 581                 that all entries in mylist have the same category and package name."""
 582                 if (mylist is None) or (len(mylist) == 0):
 583                         return []
 584                 newlist = mylist[:]
 585                 #first, we mask out packages in the package.mask file
 586                 mykey = newlist[0]
 587                 cpv = catpkgsplit(mykey)
 588                 if not cpv:
 589                         #invalid cat/pkg-v
 590                         print "visible(): invalid cat/pkg-v:", mykey
 591                         return []
 592                 mycp = cpv[0] + "/" + cpv[1]
 593                 maskdict = self.mysettings.pmaskdict
 594                 unmaskdict = self.mysettings.punmaskdict
 595                 if maskdict.has_key(mycp):
 596                         for x in maskdict[mycp]:
 597                                 mymatches = self.xmatch("match-all", x)
 598                                 if mymatches is None:
 599                                         #error in package.mask file; print warning and continue:
 600                                         print "visible(): package.mask entry \"" + x + "\" is invalid, ignoring..."
 601                                         continue
 602                                 for y in mymatches:
 603                                         unmask = 0
 604                                         if unmaskdict.has_key(mycp):
 605                                                 for z in unmaskdict[mycp]:
 606                                                         mymatches_unmask = self.xmatch("match-all",z)
 607                                                         if y in mymatches_unmask:
 608                                                                 unmask = 1
 609                                                                 break
 610                                         if unmask == 0:
 611                                                 try:
 612                                                         newlist.remove(y)
 613                                                 except ValueError:
 614                                                         pass
 615
 616                 revmaskdict = self.mysettings.prevmaskdict
 617                 if revmaskdict.has_key(mycp):
 618                         for x in revmaskdict[mycp]:
 619                                 #important: only match against the still-unmasked entries...
 620                                 #notice how we pass "newlist" to the xmatch() call below....
 621                                 #Without this, ~ deps in the packages files are broken.
 622                                 mymatches = self.xmatch("match-list", x, mylist=newlist)
 623                                 if mymatches is None:
 624                                         #error in packages file; print warning and continue:
 625                                         print "emerge: visible(): profile packages entry \""+x+"\" is invalid, ignoring..."
 626                                         continue
 627                                 pos = 0
 628                                 while pos < len(newlist):
 629                                         if newlist[pos] not in mymatches:
 630                                                 del newlist[pos]
 631                                         else:
 632                                                 pos += 1
 633                 return newlist
 634
 635         def gvisible(self,mylist):
 636                 "strip out group-masked (not in current group) entries"
 637
 638                 if mylist is None:
 639                         return []
 640                 newlist=[]
 641
 642                 accept_keywords = self.mysettings["ACCEPT_KEYWORDS"].split()
 643                 pkgdict = self.mysettings.pkeywordsdict
 644                 for mycpv in mylist:
 645                         try:
 646                                 keys, eapi = self.aux_get(mycpv, ["KEYWORDS", "EAPI"])
 647                         except KeyError:
 648                                 continue
 649                         except PortageException, e:
 650                                 writemsg("!!! Error: aux_get('%s', ['KEYWORDS', 'EAPI'])\n" % \
 651                                         mycpv, noiselevel=-1)
 652                                 writemsg("!!! %s\n" % str(e), noiselevel=-1)
 653                                 del e
 654                                 continue
 655                         mygroups = keys.split()
 656                         # Repoman may modify this attribute as necessary.
 657                         pgroups = accept_keywords[:]
 658                         match=0
 659                         cp = dep_getkey(mycpv)
 660                         if pkgdict.has_key(cp):
 661                                 matches = match_to_list(mycpv, pkgdict[cp].keys())
 662                                 for atom in matches:
 663                                         pgroups.extend(pkgdict[cp][atom])
 664                                 if matches:
 665                                         inc_pgroups = []
 666                                         for x in pgroups:
 667                                                 # The -* special case should be removed once the tree
 668                                                 # is clean of KEYWORDS=-* crap
 669                                                 if x != "-*" and x.startswith("-"):
 670                                                         try:
 671                                                                 inc_pgroups.remove(x[1:])
 672                                                         except ValueError:
 673                                                                 pass
 674                                                 if x not in inc_pgroups:
 675                                                         inc_pgroups.append(x)
 676                                         pgroups = inc_pgroups
 677                                         del inc_pgroups
 678                         hasstable = False
 679                         hastesting = False
 680                         for gp in mygroups:
 681                                 if gp=="*":
 682                                         writemsg("--- WARNING: Package '%s' uses '*' keyword.\n" % mycpv,
 683                                                 noiselevel=-1)
 684                                         match=1
 685                                         break
 686                                 elif gp in pgroups:
 687                                         match=1
 688                                         break
 689                                 elif gp[0] == "~":
 690                                         hastesting = True
 691                                 elif gp[0] != "-":
 692                                         hasstable = True
 693                         if not match and ((hastesting and "~*" in pgroups) or (hasstable and "*" in pgroups) or "**" in pgroups):
 694                                 match=1
 695                         if match and eapi_is_supported(eapi):
 696                                 newlist.append(mycpv)
 697                 return newlist
 698
 699
 700 def close_portdbapi_caches():
 701         for i in portdbapi.portdbapi_instances:
 702                 i.close_caches()
 703
 704
 705 class portagetree(object):
 706         def __init__(self, root="/", virtual=None, clone=None, settings=None):
 707                 """
 708                 Constructor for a PortageTree
 709
 710                 @param root: ${ROOT}, defaults to '/', see make.conf(5)
 711                 @type root: String/Path
 712                 @param virtual: UNUSED
 713                 @type virtual: No Idea
 714                 @param clone: Set this if you want a copy of Clone
 715                 @type clone: Existing portagetree Instance
 716                 @param settings: Portage Configuration object (portage.settings)
 717                 @type settings: Instance of portage.config
 718                 """
 719
 720                 if clone:
 721                         self.root = clone.root
 722                         self.portroot = clone.portroot
 723                         self.pkglines = clone.pkglines
 724                 else:
 725                         self.root = root
 726                         if settings is None:
 727                                 from portage import settings
 728                         self.settings = settings
 729                         self.portroot = settings["PORTDIR"]
 730                         self.virtual = virtual
 731                         self.dbapi = portdbapi(
 732                                 settings["PORTDIR"], mysettings=settings)
 733
 734         def dep_bestmatch(self,mydep):
 735                 "compatibility method"
 736                 mymatch = self.dbapi.xmatch("bestmatch-visible",mydep)
 737                 if mymatch is None:
 738                         return ""
 739                 return mymatch
 740
 741         def dep_match(self,mydep):
 742                 "compatibility method"
 743                 mymatch = self.dbapi.xmatch("match-visible",mydep)
 744                 if mymatch is None:
 745                         return []
 746                 return mymatch
 747
 748         def exists_specific(self,cpv):
 749                 return self.dbapi.cpv_exists(cpv)
 750
 751         def getallnodes(self):
 752                 """new behavior: these are all *unmasked* nodes.  There may or may not be available
 753                 masked package for nodes in this nodes list."""
 754                 return self.dbapi.cp_all()
 755
 756         def getname(self, pkgname):
 757                 "returns file location for this particular package (DEPRECATED)"
 758                 if not pkgname:
 759                         return ""
 760                 mysplit = pkgname.split("/")
 761                 psplit = pkgsplit(mysplit[1])
 762                 return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild"
 763
 764         def resolve_specific(self, myspec):
 765                 cps = catpkgsplit(myspec)
 766                 if not cps:
 767                         return None
 768                 mykey = key_expand(cps[0]+"/"+cps[1], mydb=self.dbapi,
 769                         settings=self.settings)
 770                 mykey = mykey + "-" + cps[2]
 771                 if cps[3] != "r0":
 772                         mykey = mykey + "-" + cps[3]
 773                 return mykey
 774
 775         def depcheck(self, mycheck, use="yes", myusesplit=None):
 776                 return dep_check(mycheck, self.dbapi, use=use, myuse=myusesplit)
 777
 778         def getslot(self,mycatpkg):
 779                 "Get a slot for a catpkg; assume it exists."
 780                 myslot = ""
 781                 try:
 782                         myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0]
 783                 except SystemExit, e:
 784                         raise
 785                 except Exception, e:
 786                         pass
 787                 return myslot
 788