portage.portdb.flush_cache()
- try:
- os.umask(002)
- os.chown(cachedir, os.getuid(), portage.portage_gid)
- os.chmod(cachedir, 02775)
- except SystemExit, e:
- raise # Needed else can't exit
- except:
- pass
+ ec = portage.eclass_cache.cache(portage.portdb.porttree_root)
+ # kinda ugly.
+ # XXX: nuke the filter when mr UNUSED_0? keys are dead
+ cm = portage.settings.load_best_module("portdbapi.metadbmodule")(myportdir, "metadata/cache",
+ filter(lambda x: not x.startswith("UNUSED_0"), portage.auxdbkeys))
+
# we don't make overlay trees cache here, plus we don't trust portage.settings.categories
porttree_root = portage.portdb.porttree_root
conf = portage.config(config_profile_path=portage.settings.profile_path[:], \
pass
pdb = portage.portdbapi(porttree_root, conf)
- cp_list = pdb.cp_all()
- if len(cp_list) == 0:
- print "no metadata to transfer, exiting"
- sys.exit(0)
- cp_list.sort()
- pcnt=0
- pcntstr=""
- pcntcount=len(cp_list)/100.0
- nextupdate=pcntcount
- current=1
-
- def cleanse_cache(pdb, cat, saves, porttree_root=porttree_root):
- try:
- if len(saves):
- d={}
- for v in saves:
- d[portage.catsplit(v)[1]] = True
- for pv in pdb.auxdb[porttree_root][cat].keys():
- if pv not in d:
- pdb.auxdb[porttree_root][cat].del_key(pv)
- else:
- try:
- pdb.auxdb[porttree_root][cat].clear()
- del pdb.auxdb[porttree_root][cat]
- except KeyError:
- pass
- except KeyError:
- # stop breaking things, cleansing is minor.
- pass
- savelist = []
- catlist = []
- oldcat = portage.catsplit(cp_list[0])[0]
- for cp in cp_list:
- current += 1
- if current >= nextupdate:
- pcnt += 1
- nextupdate += pcntcount
- if "--quiet" not in myopts:
- pcntstr = str(pcnt)
- sys.stdout.write("\b"*(len(pcntstr)+1)+pcntstr+"%")
- sys.stdout.flush()
- cat = portage.catsplit(cp)[0]
- if cat != oldcat:
- catlist.append(oldcat)
- cleanse_cache(pdb, oldcat, savelist)
- savelist = []
- oldcat = cat
- mymatches = pdb.xmatch("match-all", cp)
- savelist.extend(mymatches)
- for cpv in mymatches:
- try: pdb.aux_get(cpv, ["IUSE"],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
- except SystemExit: raise
- except Exception, e: print "\nFailed cache update:",cpv,e
- catlist.append(oldcat)
- catlist.append("local")
- cleanse_cache(pdb, oldcat, savelist)
- filelist = portage.listdir(cachedir+"/"+myportdir)
- for x in filelist:
- found = False
- for y in catlist:
- if x.startswith(y):
- found = True
- break
- if not found:
- portage.spawn("cd /; rm -Rf "+cachedir+"/"+myportdir+"/"+x,portage.settings,free=1,droppriv=1)
+ cp_all_list = pdb.cp_all()
+ import cache.util
+
+ class percentage_noise_maker(cache.util.quiet_mirroring):
+ def __init__(self, dbapi):
+ self.dbapi = dbapi
+ self.cp_all = dbapi.cp_all()
+ self.cp_all.sort()
+ l = len(self.cp_all)
+ self.call_update_min = 100000000
+ self.min_cp_all = l/100.0
+ self.count = 1
+ self.pstr = ''
+
+ def __iter__(self):
+ for x in self.cp_all:
+ self.count += 1
+ if self.count > self.min_cp_all:
+ self.call_update_min = 0
+ self.count = 0
+ for y in self.dbapi.cp_list(x):
+ yield y
+ self.call_update_mine = 0
+
+ def update(self, *arg):
+ try: self.pstr = int(self.pstr) + 1
+ except ValueError: self.pstr = 1
+ sys.stdout.write("%s%i%%" % ("\b" * (len(str(self.pstr))+1), self.pstr))
+ sys.stdout.flush()
+ self.call_update_min = 10000000
+
+ def finish(self, *arg):
+ sys.stdout.write("\b\b\b\b100%\n")
+ sys.stdout.flush()
- sys.stdout.write("\n\n")
+ if "--quiet" in myopts:
+ def quicky_cpv_generator(cp_all_list):
+ for x in cp_all_list:
+ for y in pdb.cp_list(x):
+ yield y
+ source = quicky_cpv_generator(pdb.cp_all())
+ noise_maker = cache.util.quiet_mirroring()
+ else:
+ noise_maker = source = percentage_noise_maker(pdb)
+ cache.util.mirror_cache(source, cm, pdb.auxdb[porttree_root], eclass_cache=ec, verbose_instance=noise_maker)
+
sys.stdout.flush()
portage.portageexit()
print "processing",x
for y in mymatches:
try:
- foo=portage.portdb.aux_get(y,["DEPEND"],debug=1)
+ foo=portage.portdb.aux_get(y,["DEPEND"])
except SystemExit, e:
# sys.exit is an exception... And consequently, we can't catch it.
raise
import commands
from time import sleep
from random import shuffle
+ from cache.cache_errors import CacheError
except SystemExit, e:
raise
except Exception, e:
from portage_locks import unlockfile,unlockdir,lockfile,lockdir
import portage_checksum
from portage_checksum import perform_md5,perform_checksum,prelink_capable
+ import eclass_cache
from portage_localization import _
except SystemExit, e:
raise
if self.modules["user"] == None:
self.modules["user"] = {}
self.modules["default"] = {
- "portdbapi.metadbmodule": "portage_db_metadata.database",
- "portdbapi.auxdbmodule": "portage_db_flat.database",
- "eclass_cache.dbmodule": "portage_db_cpickle.database",
+ "portdbapi.metadbmodule": "cache.metadata.database",
+ "portdbapi.auxdbmodule": "cache.flat_hash.database",
}
self.usemask=[]
def populate(self):
self.populated=1
-# ----------------------------------------------------------------------------
-class eclass_cache:
- """Maintains the cache information about eclasses used in ebuild."""
- def __init__(self,porttree_root,settings):
- self.porttree_root = porttree_root
- self.settings = settings
- self.depcachedir = self.settings.depcachedir[:]
-
- self.dbmodule = self.settings.load_best_module("eclass_cache.dbmodule")
-
- self.packages = {} # {"PV": {"eclass1": ["location", "_mtime_"]}}
- self.eclasses = {} # {"Name": ["location","_mtime_"]}
-
- # don't fool with porttree ordering unless you *ensure* that ebuild.sh's inherit
- # ordering is *exactly* the same
- self.porttrees=[self.porttree_root]
- self.porttrees.extend(self.settings["PORTDIR_OVERLAY"].split())
- #normalize the path now, so it's not required later.
- self.porttrees = [os.path.normpath(x) for x in self.porttrees]
- self.update_eclasses()
-
- def close_caches(self):
- for x in self.packages.keys():
- for y in self.packages[x].keys():
- try:
- self.packages[x][y].sync()
- self.packages[x][y].close()
- except SystemExit, e:
- raise
- except Exception,e:
- writemsg("Exception when closing DB: %s: %s\n" % (Exception,e))
- del self.packages[x][y]
- del self.packages[x]
-
- def flush_cache(self):
- self.packages = {}
- self.eclasses = {}
- self.update_eclasses()
-
- def update_eclasses(self):
- self.eclasses = {}
- for x in suffix_array(self.porttrees, "/eclass"):
- if x and os.path.exists(x):
- dirlist = listdir(x)
- for y in dirlist:
- if y[-len(".eclass"):]==".eclass":
- try:
- ys=y[:-len(".eclass")]
- ymtime=os.stat(x+"/"+y)[stat.ST_MTIME]
- except SystemExit, e:
- raise
- except:
- continue
- self.eclasses[ys] = [x, ymtime]
-
- def setup_package(self, location, cat, pkg):
- if not self.packages.has_key(location):
- self.packages[location] = {}
-
- if not self.packages[location].has_key(cat):
- try:
- self.packages[location][cat] = self.dbmodule(self.depcachedir+"/"+location, cat+"-eclass", [], uid, portage_gid)
- except SystemExit, e:
- raise
- except Exception, e:
- writemsg("\n!!! Failed to open the dbmodule for eclass caching.\n")
- writemsg("!!! Generally these are permission problems. Caught exception follows:\n")
- writemsg("!!! "+str(e)+"\n")
- writemsg("!!! Dirname: "+str(self.depcachedir+"/"+location)+"\n")
- writemsg("!!! Basename: "+str(cat+"-eclass")+"\n\n")
- sys.exit(123)
-
- def sync(self, location, cat, pkg):
- if self.packages[location].has_key(cat):
- self.packages[location][cat].sync()
-
- def update_package(self, location, cat, pkg, eclass_list):
- self.setup_package(location, cat, pkg)
- if not eclass_list:
- return 1
-
- data = {}
- for x in eclass_list:
- if x not in self.eclasses:
- writemsg("Eclass '%s' does not exist for '%s'\n" % (x, cat+"/"+pkg))
- return 0
- data[x] = [self.eclasses[x][0],self.eclasses[x][1]]
-
- self.packages[location][cat][pkg] = data
- self.sync(location,cat,pkg)
- return 1
-
- def is_current(self, location, cat, pkg, eclass_list):
- self.setup_package(location, cat, pkg)
-
- if not eclass_list:
- return 1
-
- if not (self.packages[location][cat].has_key(pkg) and self.packages[location][cat][pkg] and eclass_list):
- return 0
-
- myp = self.packages[location][cat][pkg]
- for x in eclass_list:
- if not (x in self.eclasses and x in myp and myp[x] == self.eclasses[x]):
- return 0
-
- return 1
-
-# ----------------------------------------------------------------------------
-
auxdbkeys=[
'DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI',
'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION',
def close_portdbapi_caches():
for i in portdbapi.portdbapi_instances:
i.close_caches()
+
+
class portdbapi(dbapi):
"""this tree will scan a portage directory located at root (passed to init)"""
portdbapi_instances = []
if self.tmpfs and not os.access(self.tmpfs, os.R_OK):
self.tmpfs = None
- self.eclassdb = eclass_cache(self.porttree_root, self.mysettings)
+ self.eclassdb = eclass_cache.cache(self.porttree_root, overlays=settings["PORTDIR_OVERLAY"].split())
self.metadb = {}
self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule")
- self.auxdb = {}
- self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule")
-
#if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
self.xcache={}
self.frozen=0
self.porttrees=[self.porttree_root]+self.mysettings["PORTDIR_OVERLAY"].split()
+ self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule")
+ self.auxdb = {}
+ # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
+ # ~harring
+ filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys)
+ for x in self.porttrees:
+ # location, label, auxdbkeys
+ self.auxdb[x] = self.auxdbmodule(portage_const.DEPCACHE_PATH, x, filtered_auxdbkeys, gid=portage_gid)
+
def close_caches(self):
for x in self.auxdb.keys():
- for y in self.auxdb[x].keys():
- self.auxdb[x][y].sync()
- self.auxdb[x][y].close()
- del self.auxdb[x][y]
- del self.auxdb[x]
- self.eclassdb.close_caches()
+ self.auxdb[x].sync()
+ self.auxdb.clear()
def flush_cache(self):
self.metadb = {}
self.auxdb = {}
- self.eclassdb.flush_cache()
def finddigest(self,mycpv):
try:
# when not found
return None, 0
- def aux_get(self,mycpv,mylist,strict=0,metacachedir=None,debug=0):
+ def aux_get(self, mycpv, mylist):
"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
cat,pkg = string.split(mycpv, "/", 1)
- if metacachedir:
- if cat not in self.metadb:
- self.metadb[cat] = self.metadbmodule(metacachedir,cat,auxdbkeys,uid,portage_gid)
-
myebuild, mylocation=self.findname2(mycpv)
if not myebuild:
raise portage_exception.SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)}
writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath})
- if mylocation not in self.auxdb:
- self.auxdb[mylocation] = {}
-
- if not self.auxdb[mylocation].has_key(cat):
- self.auxdb[mylocation][cat] = self.auxdbmodule(self.depcachedir+"/"+mylocation,cat,auxdbkeys,uid,portage_gid)
if os.access(myebuild, os.R_OK):
emtime=os.stat(myebuild)[stat.ST_MTIME]
raise KeyError
try:
- auxdb_is_valid = self.auxdb[mylocation][cat].has_key(pkg) and \
- self.auxdb[mylocation][cat][pkg].has_key("_mtime_") and \
- self.auxdb[mylocation][cat][pkg]["_mtime_"] == emtime
- except SystemExit, e:
- raise
- except Exception, e:
- auxdb_is_valid = 0
- if not metacachedir:
- writemsg("auxdb exception: [%(loc)s]: %(exception)s\n" % {"loc":mylocation+"::"+cat+"/"+pkg, "exception":str(e)})
- if self.auxdb[mylocation][cat].has_key(pkg):
- self.auxdb[mylocation][cat].del_key(pkg)
- self.auxdb[mylocation][cat].sync()
-
- writemsg("auxdb is valid: "+str(auxdb_is_valid)+" "+str(pkg)+"\n", 2)
- doregen = not (auxdb_is_valid and self.eclassdb.is_current(mylocation,cat,pkg,self.auxdb[mylocation][cat][pkg]["INHERITED"].split()))
-
- # when mylocation is not overlay directorys and metacachedir is set,
- # we use cache files, which is usually on /usr/portage/metadata/cache/.
- if doregen and mylocation==self.mysettings["PORTDIR"] and metacachedir and self.metadb[cat].has_key(pkg):
- metadata=self.metadb[cat][pkg]
-
- if "EAPI" not in metadata or not metadata["EAPI"].strip():
- metadata["EAPI"] = "0"
-
- if not eapi_is_supported(metadata["EAPI"]):
- # intentionally wipe keys.
- eapi = metadata["EAPI"]
- mtime = metadata.get("_mtime_", 0)
- metadata = {}
- map(lambda x: metadata.setdefault(x, ''), auxdbkeys)
- metadata["_mtime_"] = long(mtime)
- metadata["EAPI"] == "-"+eapi
-
+ mydata = self.auxdb[mylocation][mycpv]
+ if emtime != long(mydata.get("_mtime_", 0)):
+ doregen = True
+ elif len(mydata.get("_eclasses_", [])) > 0:
+ doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"])
else:
- # eclass updates only if we haven't nuked the entry.
- self.eclassdb.update_package(mylocation,cat,pkg,metadata["INHERITED"].split())
+ doregen = False
+
+ except KeyError:
+ doregen = True
+ except CacheError:
+ doregen = True
+ try: del self.auxdb[mylocation][mycpv]
+ except KeyError: pass
- self.auxdb[mylocation][cat][pkg] = metadata
- self.auxdb[mylocation][cat].sync()
+ writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2)
- elif doregen:
+ if doregen:
writemsg("doregen: %s %s\n" % (doregen,mycpv), 2)
writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n",1)
if os.path.exists(mydbkey):
try:
os.unlink(mydbkey)
- except SystemExit, e:
- raise
- except Exception, e:
+ except (IOError, OSError), e:
portage_locks.unlockfile(mylock)
self.lock_held = 0
writemsg("Uncaught handled exception: %(exception)s\n" % {"exception":str(e)})
os.unlink(mydbkey)
mylines=mycent.readlines()
mycent.close()
- except SystemExit, e:
- raise
+
except (IOError, OSError):
portage_locks.unlockfile(mylock)
self.lock_held = 0
writemsg(str(red("\naux_get():")+" (1) Error in "+mycpv+" ebuild.\n"
" Check for syntax error or corruption in the ebuild. (--debug)\n\n"))
raise KeyError
- except Exception, e:
- portage_locks.unlockfile(mylock)
- self.lock_held = 0
- writemsg("Uncaught handled exception: %(exception)s\n" % {"exception":str(e)})
- raise
portage_locks.unlockfile(mylock)
self.lock_held = 0
map(lambda x:mydata.setdefault(x, ""), auxdbkeys)
mydata["EAPI"] = "-"+eapi
+ if mydata.get("INHERITED", False):
+ mydata["_eclasses_"] = self.eclassdb.get_eclass_data(mydata["INHERITED"].split())
+ else:
+ mydata["_eclasses_"] = {}
+
+ del mydata["INHERITED"]
+
mydata["_mtime_"] = emtime
- self.auxdb[mylocation][cat][pkg] = mydata
- self.auxdb[mylocation][cat].sync()
- if not self.eclassdb.update_package(mylocation, cat, pkg, mylines[auxdbkeys.index("INHERITED")].split()):
- sys.exit(1)
+ self.auxdb[mylocation][mycpv] = mydata
#finally, we look at our internal cache entry and return the requested data.
- mydata = self.auxdb[mylocation][cat][pkg]
returnme = []
for x in mylist:
- returnme.append(mydata.get(x,""))
+ if x == "INHERITED":
+ returnme.append(' '.join(mydata.get("_eclasses_", {}).keys()))
+ else:
+ returnme.append(mydata.get(x,""))
if "EAPI" in mylist:
idx = mylist.index("EAPI")
projects / portage.git / commitdiff