From fb55138d38d5ef89d47ccd32c051af1e03d33f74 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 13 Feb 2007 19:51:52 +0000 Subject: [PATCH] Merged from trunk 5937:5961 (hi genone!) add docstrings (WarnerBro) Move dbapi and related classes out of __init__.py into their own subpackage. Also perform some minor cleanups. (genone) fix syntax Fix imports from the emerge package. Fix automatic sys.path insertions so that they're relative instead of absolute hard coded. remove broken dblink import fix broken imports Use os.path.realpath to handle symlinks properly. Break a recursive loop. Fix missing stat import. Fix vardb.cpv_all() breakage. Add missing VDB_PATH to fix vardbapi.aux_get() breakage. Make xterm titles work even when color is disabled. Thanks to Joachim Kaeber in bug #166255. Use the real symbols in import statements, not the references in __init__.py (genone) import best() from portage.versions Make subpackage naming consistent (genone) In the event of a circular dependency panic, eliminate root nodes in order to reduce the noise level to a minimum. tweak PORT_LOGDIR comments so as to not give the wrong impression about the default value (SpankMan) Fix missing imports. Detect potential issues with mtime granlarity in env_update() and sleep if necessary. Thanks to Brian Harring for reporting. svn path=/main/branches/prefix/; revision=5962 --- bin/archive-conf | 5 +- bin/chkcontents | 3 +- bin/clean_locks | 3 +- bin/dispatch-conf | 3 +- bin/ebuild | 3 +- bin/emaint | 3 +- bin/emerge | 12 +- bin/env-update | 3 +- bin/fix-db.py | 3 +- bin/fixpackages | 3 +- bin/md5check.py | 3 +- bin/pkgname | 3 +- bin/portageq | 3 +- bin/regenworld | 3 +- bin/repoman | 3 +- bin/xpak | 3 +- cnf/make.conf | 3 +- pym/portage/__init__.py | 3330 +---------------- pym/portage/dbapi/__init__.py | 55 + pym/portage/dbapi/bintree.py | 522 +++ pym/portage/dbapi/porttree.py | 783 ++++ pym/portage/dbapi/vartree.py | 1839 +++++++++ pym/portage/dbapi/virtual.py | 90 + pym/portage/dep.py | 20 + pym/portage/elog/__init__.py | 111 + .../{elog_modules => elog}/mod_custom.py | 0 .../{elog_modules => elog}/mod_mail.py | 2 +- .../mod_mail_summary.py | 0 .../{elog_modules => elog}/mod_save.py | 2 +- .../mod_save_summary.py | 0 .../{elog_modules => elog}/mod_syslog.py | 0 pym/portage/elog_modules/__init__.py | 0 pym/portage/output.py | 2 +- 33 files changed, 3485 insertions(+), 3333 deletions(-) create mode 100644 pym/portage/dbapi/__init__.py create mode 100644 pym/portage/dbapi/bintree.py create mode 100644 pym/portage/dbapi/porttree.py create mode 100644 pym/portage/dbapi/vartree.py create mode 100644 pym/portage/dbapi/virtual.py create mode 100644 pym/portage/elog/__init__.py rename pym/portage/{elog_modules => elog}/mod_custom.py (100%) rename pym/portage/{elog_modules => elog}/mod_mail.py (93%) rename pym/portage/{elog_modules => elog}/mod_mail_summary.py (100%) rename pym/portage/{elog_modules => elog}/mod_save.py (96%) rename pym/portage/{elog_modules => elog}/mod_save_summary.py (100%) rename pym/portage/{elog_modules => elog}/mod_syslog.py (100%) delete mode 100644 pym/portage/elog_modules/__init__.py diff --git a/bin/archive-conf b/bin/archive-conf index 92a310601..2c424355f 100755 --- a/bin/archive-conf +++ b/bin/archive-conf @@ -14,8 +14,9 @@ import os, sys try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") - import portage + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) + import portage import dispatch_conf diff --git a/bin/chkcontents b/bin/chkcontents index 90a586337..a4dec42c2 100755 --- a/bin/chkcontents +++ b/bin/chkcontents @@ -12,7 +12,8 @@ import os.path, os, sys try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage def CONTENTScheck(path): diff --git a/bin/clean_locks b/bin/clean_locks index 7717f423a..176370e5f 100755 --- a/bin/clean_locks +++ b/bin/clean_locks @@ -7,7 +7,8 @@ import os,sys,errno try: import portage.locks except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage.locks from portage.const import EPREFIX diff --git a/bin/dispatch-conf b/bin/dispatch-conf index 10dc33830..2c9f25446 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -21,7 +21,8 @@ import atexit, commands, os, re, shutil, stat, sys try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage import dispatch_conf, portage_const diff --git a/bin/ebuild b/bin/ebuild index 08c382083..e787d6ab8 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -24,7 +24,8 @@ os.environ["PORTAGE_CALLER"]="ebuild" try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage import portage.util, portage.const diff --git a/bin/emaint b/bin/emaint index a8eb397cc..41f9b27b2 100755 --- a/bin/emaint +++ b/bin/emaint @@ -7,7 +7,8 @@ import re try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage import portage.const, portage.exception diff --git a/bin/emerge b/bin/emerge index 47b269a93..f8e4f69c0 100755 --- a/bin/emerge +++ b/bin/emerge @@ -26,7 +26,8 @@ os.environ["PORTAGE_LEGACY_GLOBALS"] = "false" try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage del os.environ["PORTAGE_LEGACY_GLOBALS"] from portage import digraph, portdbapi @@ -2057,6 +2058,15 @@ class depgraph: self.altlist() print "!!! Error: circular dependencies:" print + # Reduce the noise level to a minimum via elimination of root + # nodes. + while True: + root_nodes = mygraph.root_nodes( + ignore_priority=DepPriority.SOFT) + if not root_nodes: + break + for node in root_nodes: + mygraph.remove(node) mygraph.debug_print() sys.exit(1) diff --git a/bin/env-update b/bin/env-update index 09d6b7bc7..56321b74e 100755 --- a/bin/env-update +++ b/bin/env-update @@ -26,7 +26,8 @@ if len(sys.argv) > 1: try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage try: portage.env_update(makelinks) diff --git a/bin/fix-db.py b/bin/fix-db.py index ae89e9e76..1b258694b 100755 --- a/bin/fix-db.py +++ b/bin/fix-db.py @@ -8,7 +8,8 @@ import os,sys,re try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage from stat import * diff --git a/bin/fixpackages b/bin/fixpackages index 14f911f98..9d05826cf 100755 --- a/bin/fixpackages +++ b/bin/fixpackages @@ -8,7 +8,8 @@ os.environ["PORTAGE_CALLER"]="fixpackages" try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage mysettings = portage.settings diff --git a/bin/md5check.py b/bin/md5check.py index e33640ab0..f977e764a 100755 --- a/bin/md5check.py +++ b/bin/md5check.py @@ -8,7 +8,8 @@ os.environ["FEATURES"]="mirror cvs" try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage import portage.util diff --git a/bin/pkgname b/bin/pkgname index 378e603c3..819387c58 100755 --- a/bin/pkgname +++ b/bin/pkgname @@ -7,7 +7,8 @@ import sys, os try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage a=portage.pkgsplit(sys.argv[1]) diff --git a/bin/portageq b/bin/portageq index 5d1c99902..949fd159b 100755 --- a/bin/portageq +++ b/bin/portageq @@ -346,7 +346,8 @@ def main(): try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage if uses_root: sys.argv[2] = portage.root diff --git a/bin/regenworld b/bin/regenworld index ec3c4c664..1ef2f0743 100755 --- a/bin/regenworld +++ b/bin/regenworld @@ -7,7 +7,8 @@ import sys, os try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage import re, portage_const diff --git a/bin/repoman b/bin/repoman index f60ced470..ee34f4496 100755 --- a/bin/repoman +++ b/bin/repoman @@ -26,7 +26,8 @@ os.environ["PORTAGE_LEGACY_GLOBALS"] = "false" try: import portage except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage del os.environ["PORTAGE_LEGACY_GLOBALS"] diff --git a/bin/xpak b/bin/xpak index 98fa10aee..19e66506b 100755 --- a/bin/xpak +++ b/bin/xpak @@ -11,7 +11,8 @@ if len(sys.argv)!=3: try: import portage.xpak except ImportError: - sys.path.insert(0, "@PORTAGE_BASE@/pym") + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage.xpak portage.xpak.xpak(sys.argv[1],sys.argv[2]) diff --git a/cnf/make.conf b/cnf/make.conf index 0fe7efd50..f633ba5c4 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -102,7 +102,8 @@ # providing a directory. Permissions will be modified as needed IF the # directory exists, otherwise logging will be disabled. NNNN is the # increment at the time the log is created. Logs are thus sequential. -#PORT_LOGDIR="${EPREFIX}"/var/log/portage +# The common setting is to use the /var/log/portage path. +#PORT_LOGDIR="" # # PORTDIR_OVERLAY is a directory where local ebuilds may be stored without # concern that they will be deleted by rsync updates. Default is not diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 0188e2076..1f7ff0d06 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -459,110 +459,6 @@ class digraph: print "(%s)" % self.nodes[node][0][child] -_elog_atexit_handlers = [] -def elog_process(cpv, mysettings): - mylogfiles = listdir(mysettings["T"]+"/logging/") - # shortcut for packages without any messages - if len(mylogfiles) == 0: - return - # exploit listdir() file order so we process log entries in chronological order - mylogfiles.reverse() - all_logentries = {} - for f in mylogfiles: - msgfunction, msgtype = f.split(".") - if msgfunction not in portage.const.EBUILD_PHASES: - writemsg("!!! can't process invalid log file: %s\n" % f, - noiselevel=-1) - continue - if not msgfunction in all_logentries: - all_logentries[msgfunction] = [] - msgcontent = open(mysettings["T"]+"/logging/"+f, "r").readlines() - all_logentries[msgfunction].append((msgtype, msgcontent)) - - def filter_loglevels(logentries, loglevels): - # remove unwanted entries from all logentries - rValue = {} - loglevels = map(str.upper, loglevels) - for phase in logentries.keys(): - for msgtype, msgcontent in logentries[phase]: - if msgtype.upper() in loglevels or "*" in loglevels: - if not rValue.has_key(phase): - rValue[phase] = [] - rValue[phase].append((msgtype, msgcontent)) - return rValue - - my_elog_classes = set(mysettings.get("PORTAGE_ELOG_CLASSES", "").split()) - default_logentries = filter_loglevels(all_logentries, my_elog_classes) - - # in case the filters matched all messages and no module overrides exist - if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")): - return - - def combine_logentries(logentries): - # generate a single string with all log messages - rValue = "" - for phase in portage.const.EBUILD_PHASES: - if not phase in logentries: - continue - for msgtype,msgcontent in logentries[phase]: - rValue += "%s: %s\n" % (msgtype, phase) - for line in msgcontent: - rValue += line - rValue += "\n" - return rValue - - default_fulllog = combine_logentries(default_logentries) - - # pass the processing to the individual modules - logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split() - for s in logsystems: - # allow per module overrides of PORTAGE_ELOG_CLASSES - if ":" in s: - s, levels = s.split(":", 1) - levels = levels.split(",") - mod_logentries = filter_loglevels(all_logentries, levels) - mod_fulllog = combine_logentries(mod_logentries) - else: - mod_logentries = default_logentries - mod_fulllog = default_fulllog - if len(mod_logentries) == 0: - continue - # - is nicer than _ for module names, so allow people to use it. - s = s.replace("-", "_") - try: - # FIXME: ugly ad.hoc import code - # TODO: implement a common portage module loader - logmodule = __import__("portage.elog_modules.mod_"+s) - m = getattr(logmodule, "mod_"+s) - def timeout_handler(signum, frame): - raise portage.exception.PortageException( - "Timeout in elog_process for system '%s'" % s) - import signal - signal.signal(signal.SIGALRM, timeout_handler) - # Timeout after one minute (in case something like the mail - # module gets hung). - signal.alarm(60) - try: - m.process(mysettings, cpv, mod_logentries, mod_fulllog) - finally: - signal.alarm(0) - if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers: - _elog_atexit_handlers.append(m.finalize) - atexit_register(m.finalize, mysettings) - except (ImportError, AttributeError), e: - writemsg("!!! Error while importing logging modules " + \ - "while loading \"mod_%s\":\n" % str(s)) - writemsg("%s\n" % str(e), noiselevel=-1) - except portage.exception.PortageException, e: - writemsg("%s\n" % str(e), noiselevel=-1) - - # clean logfiles to avoid repetitions - for f in mylogfiles: - try: - os.unlink(os.path.join(mysettings["T"], "logging", f)) - except OSError: - pass - #parse /etc/env.d and generate /etc/profile.env def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None): @@ -701,12 +597,18 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None): newprelink.write("-b "+x+"\n") newprelink.close() + # Portage stores mtimes with 1 second granularity but in >=python-2.5 finer + # granularity is possible. In order to avoid the potential ambiguity of + # mtimes that differ by less than 1 second, sleep here if any of the + # directories have been modified during the current second. + sleep_for_mtime_granularity = False + current_time = long(time.time()) mtime_changed = False lib_dirs = set() for lib_dir in portage.util.unique_array(specials["LDPATH"]+['usr/lib','usr/lib64','usr/lib32','lib','lib64','lib32']): x = os.path.join(target_root + EPREFIX, lib_dir.lstrip(os.sep)) try: - newldpathtime = os.stat(x)[stat.ST_MTIME] + newldpathtime = long(os.stat(x).st_mtime) lib_dirs.add(normalize_path(x)) except OSError, oe: if oe.errno == errno.ENOENT: @@ -717,6 +619,8 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None): # ignore this path because it doesn't exist continue raise + if newldpathtime == current_time: + sleep_for_mtime_granularity = True if x in prev_mtimes: if prev_mtimes[x] == newldpathtime: pass @@ -788,6 +692,10 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None): outfile.write("setenv %s '%s'\n" % (x, env[x])) outfile.close() + if sleep_for_mtime_granularity: + while current_time == long(time.time()): + sleep(1) + def ExtractKernelVersion(base_dir): """ Try to figure out what kernel version we are running @@ -4653,955 +4561,6 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): rValue.append(kmask+" keyword") return rValue -class portagetree: - def __init__(self, root="/", virtual=None, clone=None, settings=None): - """ - Constructor for a PortageTree - - @param root: ${ROOT}, defaults to '/', see make.conf(5) - @type root: String/Path - @param virtual: UNUSED - @type virtual: No Idea - @param clone: Set this if you want a copy of Clone - @type clone: Existing portagetree Instance - @param settings: Portage Configuration object (portage.settings) - @type settings: Instance of portage.config - """ - - if clone: - self.root = clone.root - self.portroot = clone.portroot - self.pkglines = clone.pkglines - else: - self.root = root - if settings is None: - settings = globals()["settings"] - self.settings = settings - self.portroot = settings["PORTDIR"] - self.virtual = virtual - self.dbapi = portdbapi( - settings["PORTDIR"], mysettings=settings) - - def dep_bestmatch(self,mydep): - "compatibility method" - mymatch=self.dbapi.xmatch("bestmatch-visible",mydep) - if mymatch is None: - return "" - return mymatch - - def dep_match(self,mydep): - "compatibility method" - mymatch=self.dbapi.xmatch("match-visible",mydep) - if mymatch is None: - return [] - return mymatch - - def exists_specific(self,cpv): - return self.dbapi.cpv_exists(cpv) - - def getallnodes(self): - """new behavior: these are all *unmasked* nodes. There may or may not be available - masked package for nodes in this nodes list.""" - return self.dbapi.cp_all() - - def getname(self,pkgname): - "returns file location for this particular package (DEPRECATED)" - if not pkgname: - return "" - mysplit=pkgname.split("/") - psplit=pkgsplit(mysplit[1]) - return self.portroot+"/"+mysplit[0]+"/"+psplit[0]+"/"+mysplit[1]+".ebuild" - - def resolve_specific(self,myspec): - cps=catpkgsplit(myspec) - if not cps: - return None - mykey = key_expand(cps[0]+"/"+cps[1], mydb=self.dbapi, - settings=self.settings) - mykey=mykey+"-"+cps[2] - if cps[3]!="r0": - mykey=mykey+"-"+cps[3] - return mykey - - def depcheck(self,mycheck,use="yes",myusesplit=None): - return dep_check(mycheck,self.dbapi,use=use,myuse=myusesplit) - - def getslot(self,mycatpkg): - "Get a slot for a catpkg; assume it exists." - myslot = "" - try: - myslot=self.dbapi.aux_get(mycatpkg,["SLOT"])[0] - except SystemExit, e: - raise - except Exception, e: - pass - return myslot - - -class dbapi: - def __init__(self): - pass - - def close_caches(self): - pass - - def cp_list(self,cp,use_cache=1): - return - - def cpv_all(self): - cpv_list = [] - for cp in self.cp_all(): - cpv_list.extend(self.cp_list(cp)) - return cpv_list - - def aux_get(self,mycpv,mylist): - "stub code for returning auxiliary db information, such as SLOT, DEPEND, etc." - 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' - 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or [] if mycpv not found' - raise NotImplementedError - - def match(self,origdep,use_cache=1): - mydep = dep_expand(origdep, mydb=self, settings=self.settings) - mykey=dep_getkey(mydep) - mylist = match_from_list(mydep,self.cp_list(mykey,use_cache=use_cache)) - myslot = portage.dep.dep_getslot(mydep) - if myslot is not None: - mylist = [cpv for cpv in mylist \ - if self.aux_get(cpv, ["SLOT"])[0] == myslot] - return mylist - - def match2(self,mydep,mykey,mylist): - writemsg("DEPRECATED: dbapi.match2\n") - match_from_list(mydep,mylist) - - def invalidentry(self, mypath): - if re.search("portage_lockfile$",mypath): - if not os.environ.has_key("PORTAGE_MASTER_PID"): - writemsg("Lockfile removed: %s\n" % mypath, 1) - portage.locks.unlockfile((mypath,None,None)) - else: - # Nothing we can do about it. We're probably sandboxed. - pass - elif re.search(".*/-MERGING-(.*)",mypath): - if os.path.exists(mypath): - writemsg(red("INCOMPLETE MERGE:")+" "+mypath+"\n", noiselevel=-1) - else: - writemsg("!!! Invalid db entry: %s\n" % mypath, noiselevel=-1) - - - -class fakedbapi(dbapi): - "This is a dbapi to use for the emptytree function. It's empty, but things can be added to it." - def __init__(self, settings=None): - self.cpvdict={} - self.cpdict={} - if settings is None: - settings = globals()["settings"] - self.settings = settings - self._match_cache = {} - - def _clear_cache(self): - if self._match_cache: - self._match_cache = {} - - def match(self, origdep, use_cache=1): - result = self._match_cache.get(origdep, None) - if result is not None: - return result[:] - result = dbapi.match(self, origdep, use_cache=use_cache) - self._match_cache[origdep] = result - return result[:] - - def cpv_exists(self,mycpv): - return self.cpvdict.has_key(mycpv) - - def cp_list(self,mycp,use_cache=1): - if not self.cpdict.has_key(mycp): - return [] - else: - return self.cpdict[mycp] - - def cp_all(self): - returnme=[] - for x in self.cpdict.keys(): - returnme.extend(self.cpdict[x]) - return returnme - - def cpv_all(self): - return self.cpvdict.keys() - - def cpv_inject(self, mycpv, metadata=None): - """Adds a cpv from the list of available packages.""" - self._clear_cache() - mycp=cpv_getkey(mycpv) - self.cpvdict[mycpv] = metadata - myslot = None - if metadata: - myslot = metadata.get("SLOT", None) - if myslot and mycp in self.cpdict: - # If necessary, remove another package in the same SLOT. - for cpv in self.cpdict[mycp]: - if mycpv != cpv: - other_metadata = self.cpvdict[cpv] - if other_metadata: - if myslot == other_metadata.get("SLOT", None): - self.cpv_remove(cpv) - break - if mycp not in self.cpdict: - self.cpdict[mycp] = [] - if not mycpv in self.cpdict[mycp]: - self.cpdict[mycp].append(mycpv) - - def cpv_remove(self,mycpv): - """Removes a cpv from the list of available packages.""" - self._clear_cache() - mycp=cpv_getkey(mycpv) - if self.cpvdict.has_key(mycpv): - del self.cpvdict[mycpv] - if not self.cpdict.has_key(mycp): - return - while mycpv in self.cpdict[mycp]: - del self.cpdict[mycp][self.cpdict[mycp].index(mycpv)] - if not len(self.cpdict[mycp]): - del self.cpdict[mycp] - - def aux_get(self, mycpv, wants): - if not self.cpv_exists(mycpv): - raise KeyError(mycpv) - metadata = self.cpvdict[mycpv] - if not metadata: - return ["" for x in wants] - return [metadata.get(x, "") for x in wants] - - def aux_update(self, cpv, values): - self._clear_cache() - self.cpvdict[cpv].update(values) - -class bindbapi(fakedbapi): - def __init__(self, mybintree=None, settings=None): - self.bintree = mybintree - self.cpvdict={} - self.cpdict={} - if settings is None: - settings = globals()["settings"] - self.settings = settings - self._match_cache = {} - # Selectively cache metadata in order to optimize dep matching. - self._aux_cache_keys = set(["SLOT"]) - self._aux_cache = {} - - def match(self, *pargs, **kwargs): - if self.bintree and not self.bintree.populated: - self.bintree.populate() - return fakedbapi.match(self, *pargs, **kwargs) - - def aux_get(self,mycpv,wants): - if self.bintree and not self.bintree.populated: - self.bintree.populate() - cache_me = False - if not set(wants).difference(self._aux_cache_keys): - aux_cache = self._aux_cache.get(mycpv) - if aux_cache is not None: - return [aux_cache[x] for x in wants] - cache_me = True - mysplit = mycpv.split("/") - mylist = [] - tbz2name = mysplit[1]+".tbz2" - if self.bintree and not self.bintree.isremote(mycpv): - tbz2 = portage.xpak.tbz2(self.bintree.getname(mycpv)) - getitem = tbz2.getfile - else: - getitem = self.bintree.remotepkgs[tbz2name].get - mydata = {} - mykeys = wants - if cache_me: - mykeys = self._aux_cache_keys.union(wants) - for x in mykeys: - myval = getitem(x) - # myval is None if the key doesn't exist - # or the tbz2 is corrupt. - if myval: - mydata[x] = " ".join(myval.split()) - if "EAPI" in mykeys: - if not mydata.setdefault("EAPI", "0"): - mydata["EAPI"] = "0" - if cache_me: - aux_cache = {} - for x in self._aux_cache_keys: - aux_cache[x] = mydata.get(x, "") - self._aux_cache[mycpv] = aux_cache - return [mydata.get(x, "") for x in wants] - - def aux_update(self, cpv, values): - if not self.bintree.populated: - self.bintree.populate() - tbz2path = self.bintree.getname(cpv) - if not os.path.exists(tbz2path): - raise KeyError(cpv) - mytbz2 = portage.xpak.tbz2(tbz2path) - mydata = mytbz2.get_data() - mydata.update(values) - mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) - - def cp_list(self, *pargs, **kwargs): - if not self.bintree.populated: - self.bintree.populate() - return fakedbapi.cp_list(self, *pargs, **kwargs) - - def cpv_all(self): - if not self.bintree.populated: - self.bintree.populate() - return fakedbapi.cpv_all(self) - -class vardbapi(dbapi): - def __init__(self, root, categories=None, settings=None, vartree=None): - self.root = root[:] - #cache for category directory mtimes - self.mtdircache = {} - #cache for dependency checks - self.matchcache = {} - #cache for cp_list results - self.cpcache = {} - self.blockers = None - if settings is None: - settings = globals()["settings"] - self.settings = settings - if categories is None: - categories = settings.categories - self.categories = categories[:] - if vartree is None: - vartree = globals()["db"][root]["vartree"] - self.vartree = vartree - self._aux_cache_keys = set(["SLOT", "COUNTER", "PROVIDE", "USE", - "IUSE", "DEPEND", "RDEPEND", "PDEPEND"]) - self._aux_cache = None - self._aux_cache_version = "1" - self._aux_cache_filename = os.path.join(self.root, - CACHE_PATH.lstrip(os.path.sep), "vdb_metadata.pickle") - - def cpv_exists(self,mykey): - "Tells us whether an actual ebuild exists on disk (no masking)" - return os.path.exists(self.root+VDB_PATH+"/"+mykey) - - def cpv_counter(self,mycpv): - "This method will grab the COUNTER. Returns a counter value." - try: - return long(self.aux_get(mycpv, ["COUNTER"])[0]) - except KeyError, ValueError: - pass - cdir=self.root+VDB_PATH+"/"+mycpv - cpath=self.root+VDB_PATH+"/"+mycpv+"/COUNTER" - - # We write our new counter value to a new file that gets moved into - # place to avoid filesystem corruption on XFS (unexpected reboot.) - corrupted=0 - if os.path.exists(cpath): - cfile=open(cpath, "r") - try: - counter=long(cfile.readline()) - except ValueError: - print "portage: COUNTER for",mycpv,"was corrupted; resetting to value of 0" - counter=long(0) - corrupted=1 - cfile.close() - elif os.path.exists(cdir): - mys = pkgsplit(mycpv) - myl = self.match(mys[0],use_cache=0) - print mys,myl - if len(myl) == 1: - try: - # Only one package... Counter doesn't matter. - write_atomic(cpath, "1") - counter = 1 - except SystemExit, e: - raise - except Exception, e: - writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n", - noiselevel=-1) - writemsg("!!! Please run /usr/lib/portage/bin/fix-db.py or\n", - noiselevel=-1) - writemsg("!!! unmerge this exact version.\n", noiselevel=-1) - writemsg("!!! %s\n" % e, noiselevel=-1) - sys.exit(1) - else: - writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n", - noiselevel=-1) - writemsg("!!! Please run /usr/lib/portage/bin/fix-db.py or\n", - noiselevel=-1) - writemsg("!!! remerge the package.\n", noiselevel=-1) - sys.exit(1) - else: - counter=long(0) - if corrupted: - # update new global counter file - write_atomic(cpath, str(counter)) - return counter - - def cpv_inject(self,mycpv): - "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" - os.makedirs(self.root+VDB_PATH+"/"+mycpv) - counter = self.counter_tick(self.root, mycpv=mycpv) - # write local package counter so that emerge clean does the right thing - write_atomic(os.path.join(self.root, VDB_PATH, mycpv, "COUNTER"), str(counter)) - - def isInjected(self,mycpv): - if self.cpv_exists(mycpv): - if os.path.exists(self.root+VDB_PATH+"/"+mycpv+"/INJECTED"): - return True - if not os.path.exists(self.root+VDB_PATH+"/"+mycpv+"/CONTENTS"): - return True - return False - - def move_ent(self,mylist): - origcp=mylist[1] - newcp=mylist[2] - - # sanity check - for cp in [origcp,newcp]: - if not (isvalidatom(cp) and isjustname(cp)): - raise portage.exception.InvalidPackageName(cp) - origmatches=self.match(origcp,use_cache=0) - if not origmatches: - return - for mycpv in origmatches: - mycpsplit=catpkgsplit(mycpv) - mynewcpv=newcp+"-"+mycpsplit[2] - mynewcat=newcp.split("/")[0] - if mycpsplit[3]!="r0": - mynewcpv += "-"+mycpsplit[3] - mycpsplit_new = catpkgsplit(mynewcpv) - origpath=self.root+VDB_PATH+"/"+mycpv - if not os.path.exists(origpath): - continue - writemsg_stdout("@") - if not os.path.exists(self.root+VDB_PATH+"/"+mynewcat): - #create the directory - os.makedirs(self.root+VDB_PATH+"/"+mynewcat) - newpath=self.root+VDB_PATH+"/"+mynewcpv - if os.path.exists(newpath): - #dest already exists; keep this puppy where it is. - continue - os.rename(origpath, newpath) - - # We need to rename the ebuild now. - old_pf = catsplit(mycpv)[1] - new_pf = catsplit(mynewcpv)[1] - if new_pf != old_pf: - try: - os.rename(os.path.join(newpath, old_pf + ".ebuild"), - os.path.join(newpath, new_pf + ".ebuild")) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e - write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") - - write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") - fixdbentries([mylist], newpath) - - def update_ents(self, update_iter): - """Run fixdbentries on all installed packages (time consuming). Like - fixpackages, this should be run from a helper script and display - a progress indicator.""" - dbdir = os.path.join(self.root, VDB_PATH) - for catdir in listdir(dbdir): - catdir = dbdir+"/"+catdir - if os.path.isdir(catdir): - for pkgdir in listdir(catdir): - pkgdir = catdir+"/"+pkgdir - if os.path.isdir(pkgdir): - fixdbentries(update_iter, pkgdir) - - def move_slot_ent(self,mylist): - pkg=mylist[1] - origslot=mylist[2] - newslot=mylist[3] - - if not isvalidatom(pkg): - raise portage.exception.InvalidAtom(pkg) - - origmatches=self.match(pkg,use_cache=0) - - if not origmatches: - return - for mycpv in origmatches: - origpath=self.root+VDB_PATH+"/"+mycpv - if not os.path.exists(origpath): - continue - - slot=grabfile(origpath+"/SLOT"); - if (not slot): - continue - - if (slot[0]!=origslot): - continue - - writemsg_stdout("s") - write_atomic(os.path.join(origpath, "SLOT"), newslot+"\n") - - def cp_list(self,mycp,use_cache=1): - mysplit=mycp.split("/") - if mysplit[0] == '*': - mysplit[0] = mysplit[0][1:] - try: - mystat=os.stat(self.root+VDB_PATH+"/"+mysplit[0])[stat.ST_MTIME] - except OSError: - mystat=0 - if use_cache and self.cpcache.has_key(mycp): - cpc=self.cpcache[mycp] - if cpc[0]==mystat: - return cpc[1] - list=listdir(self.root+VDB_PATH+"/"+mysplit[0],EmptyOnError=1) - - if (list is None): - return [] - returnme=[] - for x in list: - if x.startswith("."): - continue - if x[0] == '-': - #writemsg(red("INCOMPLETE MERGE:")+str(x[len("-MERGING-"):])+"\n") - continue - ps=pkgsplit(x) - if not ps: - self.invalidentry(self.root+VDB_PATH+"/"+mysplit[0]+"/"+x) - continue - if len(mysplit) > 1: - if ps[0]==mysplit[1]: - returnme.append(mysplit[0]+"/"+x) - if use_cache: - self.cpcache[mycp]=[mystat,returnme] - elif self.cpcache.has_key(mycp): - del self.cpcache[mycp] - return returnme - - def cpv_all(self,use_cache=1): - returnme=[] - basepath = self.root+VDB_PATH+"/" - - for x in self.categories: - for y in listdir(basepath+x,EmptyOnError=1): - if y.startswith("."): - continue - subpath = x+"/"+y - # -MERGING- should never be a cpv, nor should files. - if os.path.isdir(basepath+subpath) and (pkgsplit(y) is not None): - returnme += [subpath] - return returnme - - def cp_all(self,use_cache=1): - mylist = self.cpv_all(use_cache=use_cache) - d={} - for y in mylist: - if y[0] == '*': - y = y[1:] - mysplit=catpkgsplit(y) - if not mysplit: - self.invalidentry(self.root+VDB_PATH+"/"+y) - continue - d[mysplit[0]+"/"+mysplit[1]] = None - return d.keys() - - def checkblockers(self,origdep): - pass - - def match(self,origdep,use_cache=1): - "caching match function" - mydep = dep_expand( - origdep, mydb=self, use_cache=use_cache, settings=self.settings) - mykey=dep_getkey(mydep) - mycat=mykey.split("/")[0] - if not use_cache: - if self.matchcache.has_key(mycat): - del self.mtdircache[mycat] - del self.matchcache[mycat] - mymatch = match_from_list(mydep, - self.cp_list(mykey, use_cache=use_cache)) - myslot = portage.dep.dep_getslot(mydep) - if myslot is not None: - mymatch = [cpv for cpv in mymatch \ - if self.aux_get(cpv, ["SLOT"])[0] == myslot] - return mymatch - try: - curmtime=os.stat(self.root+VDB_PATH+"/"+mycat)[stat.ST_MTIME] - except (IOError, OSError): - curmtime=0 - - if not self.matchcache.has_key(mycat) or not self.mtdircache[mycat]==curmtime: - # clear cache entry - self.mtdircache[mycat]=curmtime - self.matchcache[mycat]={} - if not self.matchcache[mycat].has_key(mydep): - mymatch=match_from_list(mydep,self.cp_list(mykey,use_cache=use_cache)) - myslot = portage.dep.dep_getslot(mydep) - if myslot is not None: - mymatch = [cpv for cpv in mymatch \ - if self.aux_get(cpv, ["SLOT"])[0] == myslot] - self.matchcache[mycat][mydep]=mymatch - return self.matchcache[mycat][mydep][:] - - def findname(self, mycpv): - return self.root+VDB_PATH+"/"+str(mycpv)+"/"+mycpv.split("/")[1]+".ebuild" - - def flush_cache(self): - """If the current user has permission and the internal aux_get cache has - been updated, save it to disk and mark it unmodified. This is called - by emerge after it has loaded the full vdb for use in dependency - calculations. Currently, the cache is only written if the user has - superuser privileges (since that's required to obtain a lock), but all - users have read access and benefit from faster metadata lookups (as - long as at least part of the cache is still valid).""" - if self._aux_cache is not None and \ - self._aux_cache["modified"] and \ - secpass >= 2: - valid_nodes = set(self.cpv_all()) - for cpv in self._aux_cache["packages"].keys(): - if cpv not in valid_nodes: - del self._aux_cache["packages"][cpv] - del self._aux_cache["modified"] - try: - f = atomic_ofstream(self._aux_cache_filename) - cPickle.dump(self._aux_cache, f, -1) - f.close() - portage.util.apply_secpass_permissions( - self._aux_cache_filename, gid=portage_gid, mode=0644) - except (IOError, OSError), e: - pass - self._aux_cache["modified"] = False - - def aux_get(self, mycpv, wants): - """This automatically caches selected keys that are frequently needed - by emerge for dependency calculations. The cached metadata is - considered valid if the mtime of the package directory has not changed - since the data was cached. The cache is stored in a pickled dict - object with the following format: - - {version:"1", "packages":{cpv1:(mtime,{k1,v1, k2,v2, ...}), cpv2...}} - - If an error occurs while loading the cache pickle or the version is - unrecognized, the cache will simple be recreated from scratch (it is - completely disposable). - """ - if not self._aux_cache_keys.intersection(wants): - return self._aux_get(mycpv, wants) - if self._aux_cache is None: - try: - f = open(self._aux_cache_filename) - mypickle = cPickle.Unpickler(f) - mypickle.find_global = None - self._aux_cache = mypickle.load() - f.close() - del f - except (IOError, OSError, EOFError, cPickle.UnpicklingError): - pass - if not self._aux_cache or \ - not isinstance(self._aux_cache, dict) or \ - self._aux_cache.get("version") != self._aux_cache_version or \ - not self._aux_cache.get("packages"): - self._aux_cache = {"version":self._aux_cache_version} - self._aux_cache["packages"] = {} - self._aux_cache["modified"] = False - mydir = os.path.join(self.root, VDB_PATH, mycpv) - mydir_stat = None - try: - mydir_stat = os.stat(mydir) - except OSError, e: - if e.errno != errno.ENOENT: - raise - raise KeyError(mycpv) - mydir_mtime = long(mydir_stat.st_mtime) - pkg_data = self._aux_cache["packages"].get(mycpv) - mydata = {} - cache_valid = False - if pkg_data: - cache_mtime, metadata = pkg_data - cache_valid = cache_mtime == mydir_mtime - if cache_valid and set(metadata) != self._aux_cache_keys: - # Allow self._aux_cache_keys to change without a cache version - # bump. - cache_valid = False - if cache_valid: - mydata.update(metadata) - pull_me = set(wants).difference(self._aux_cache_keys) - else: - pull_me = self._aux_cache_keys.union(wants) - if pull_me: - # pull any needed data and cache it - aux_keys = list(pull_me) - for k, v in izip(aux_keys, self._aux_get(mycpv, aux_keys)): - mydata[k] = v - if not cache_valid: - cache_data = {} - for aux_key in self._aux_cache_keys: - cache_data[aux_key] = mydata[aux_key] - self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) - self._aux_cache["modified"] = True - return [mydata[x] for x in wants] - - def _aux_get(self, mycpv, wants): - mydir = os.path.join(self.root, VDB_PATH, mycpv) - try: - if not stat.S_ISDIR(os.stat(mydir).st_mode): - raise KeyError(mycpv) - except OSError, e: - if e.errno == errno.ENOENT: - raise KeyError(mycpv) - del e - raise - results = [] - for x in wants: - try: - myf = open(os.path.join(mydir, x), "r") - try: - myd = myf.read() - finally: - myf.close() - myd = " ".join(myd.split()) - except IOError: - myd = "" - if x == "EAPI" and not myd: - results.append("0") - else: - results.append(myd) - return results - - def aux_update(self, cpv, values): - cat, pkg = cpv.split("/") - mylink = dblink(cat, pkg, self.root, self.settings, - treetype="vartree", vartree=self.vartree) - if not mylink.exists(): - raise KeyError(cpv) - for k, v in values.iteritems(): - mylink.setfile(k, v) - - def counter_tick(self,myroot,mycpv=None): - return self.counter_tick_core(myroot,incrementing=1,mycpv=mycpv) - - def get_counter_tick_core(self,myroot,mycpv=None): - return self.counter_tick_core(myroot,incrementing=0,mycpv=mycpv)+1 - - def counter_tick_core(self,myroot,incrementing=1,mycpv=None): - "This method will grab the next COUNTER value and record it back to the global file. Returns new counter value." - cpath=os.path.join(myroot+EPREFIX, "var/cache/edb/counter") - changed=0 - min_counter = 0 - if mycpv: - mysplit = pkgsplit(mycpv) - for x in self.match(mysplit[0],use_cache=0): - if x==mycpv: - continue - try: - old_counter = long(self.aux_get(x,["COUNTER"])[0]) - writemsg("COUNTER '%d' '%s'\n" % (old_counter, x),1) - except (ValueError, KeyError): # valueError from long(), KeyError from aux_get - old_counter = 0 - writemsg("!!! BAD COUNTER in '%s'\n" % (x), noiselevel=-1) - if old_counter > min_counter: - min_counter = old_counter - - # We write our new counter value to a new file that gets moved into - # place to avoid filesystem corruption. - find_counter = ("find '%s' -type f -name COUNTER | " + \ - "while read f; do echo $(<\"${f}\"); done | " + \ - "sort -n | tail -n1") % os.path.join(self.root, VDB_PATH) - if os.path.exists(cpath): - cfile=open(cpath, "r") - try: - counter=long(cfile.readline()) - except (ValueError,OverflowError): - try: - counter = long(commands.getoutput(find_counter).strip()) - writemsg("!!! COUNTER was corrupted; resetting to value of %d\n" % counter, - noiselevel=-1) - changed=1 - except (ValueError,OverflowError): - writemsg("!!! COUNTER data is corrupt in pkg db. The values need to be\n", - noiselevel=-1) - writemsg("!!! corrected/normalized so that portage can operate properly.\n", - noiselevel=-1) - writemsg("!!! A simple solution is not yet available so try #gentoo on IRC.\n") - sys.exit(2) - cfile.close() - else: - try: - counter = long(commands.getoutput(find_counter).strip()) - writemsg("!!! Global counter missing. Regenerated from counter files to: %s\n" % counter, - noiselevel=-1) - except ValueError: # Value Error for long(), probably others for commands.getoutput - writemsg("!!! Initializing global counter.\n", noiselevel=-1) - counter=long(0) - changed=1 - - if counter < min_counter: - counter = min_counter+1000 - changed = 1 - - if incrementing or changed: - - #increment counter - counter += 1 - # update new global counter file - write_atomic(cpath, str(counter)) - return counter - -class vartree(object): - "this tree will scan a var/db/pkg database located at root (passed to init)" - def __init__(self, root="/", virtual=None, clone=None, categories=None, - settings=None): - if clone: - self.root = clone.root[:] - self.dbapi = copy.deepcopy(clone.dbapi) - self.populated = 1 - self.settings = config(clone=clone.settings) - else: - self.root = root[:] - if settings is None: - settings = globals()["settings"] - self.settings = settings # for key_expand calls - if categories is None: - categories = settings.categories - self.dbapi = vardbapi(self.root, categories=categories, - settings=settings, vartree=self) - self.populated = 1 - - def zap(self,mycpv): - return - - def inject(self,mycpv): - return - - def get_provide(self,mycpv): - myprovides=[] - mylines = None - try: - mylines, myuse = self.dbapi.aux_get(mycpv, ["PROVIDE","USE"]) - if mylines: - myuse = myuse.split() - mylines = flatten(portage.dep.use_reduce(portage.dep.paren_reduce(mylines), uselist=myuse)) - for myprovide in mylines: - mys = catpkgsplit(myprovide) - if not mys: - mys = myprovide.split("/") - myprovides += [mys[0] + "/" + mys[1]] - return myprovides - except SystemExit, e: - raise - except Exception, e: - mydir = os.path.join(self.root, VDB_PATH, mycpv) - writemsg("\nParse Error reading PROVIDE and USE in '%s'\n" % mydir, - noiselevel=-1) - if mylines: - writemsg("Possibly Invalid: '%s'\n" % str(mylines), - noiselevel=-1) - writemsg("Exception: %s\n\n" % str(e), noiselevel=-1) - return [] - - def get_all_provides(self): - myprovides = {} - for node in self.getallcpv(): - for mykey in self.get_provide(node): - if myprovides.has_key(mykey): - myprovides[mykey] += [node] - else: - myprovides[mykey] = [node] - return myprovides - - def dep_bestmatch(self,mydep,use_cache=1): - "compatibility method -- all matches, not just visible ones" - #mymatch=best(match(dep_expand(mydep,self.dbapi),self.dbapi)) - mymatch = best(self.dbapi.match( - dep_expand(mydep, mydb=self.dbapi, settings=self.settings), - use_cache=use_cache)) - if mymatch is None: - return "" - else: - return mymatch - - def dep_match(self,mydep,use_cache=1): - "compatibility method -- we want to see all matches, not just visible ones" - #mymatch=match(mydep,self.dbapi) - mymatch=self.dbapi.match(mydep,use_cache=use_cache) - if mymatch is None: - return [] - else: - return mymatch - - def exists_specific(self,cpv): - return self.dbapi.cpv_exists(cpv) - - def getallcpv(self): - """temporary function, probably to be renamed --- Gets a list of all - category/package-versions installed on the system.""" - return self.dbapi.cpv_all() - - def getallnodes(self): - """new behavior: these are all *unmasked* nodes. There may or may not be available - masked package for nodes in this nodes list.""" - return self.dbapi.cp_all() - - def exists_specific_cat(self,cpv,use_cache=1): - cpv = key_expand(cpv, mydb=self.dbapi, use_cache=use_cache, - settings=self.settings) - a=catpkgsplit(cpv) - if not a: - return 0 - mylist=listdir(self.root+VDB_PATH+"/"+a[0],EmptyOnError=1) - for x in mylist: - b=pkgsplit(x) - if not b: - self.dbapi.invalidentry(self.root+VDB_PATH+"/"+a[0]+"/"+x) - continue - if a[1]==b[0]: - return 1 - return 0 - - def getebuildpath(self,fullpackage): - cat,package=fullpackage.split("/") - return self.root+VDB_PATH+"/"+fullpackage+"/"+package+".ebuild" - - def getnode(self,mykey,use_cache=1): - mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, - settings=self.settings) - if not mykey: - return [] - mysplit=mykey.split("/") - mydirlist=listdir(self.root+VDB_PATH+"/"+mysplit[0],EmptyOnError=1) - returnme=[] - for x in mydirlist: - mypsplit=pkgsplit(x) - if not mypsplit: - self.dbapi.invalidentry(self.root+VDB_PATH+"/"+mysplit[0]+"/"+x) - continue - if mypsplit[0]==mysplit[1]: - appendme=[mysplit[0]+"/"+x,[mysplit[0],mypsplit[0],mypsplit[1],mypsplit[2]]] - returnme.append(appendme) - return returnme - - - def getslot(self,mycatpkg): - "Get a slot for a catpkg; assume it exists." - try: - return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] - except KeyError: - return "" - - def hasnode(self,mykey,use_cache): - """Does the particular node (cat/pkg key) exist?""" - mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, - settings=self.settings) - mysplit=mykey.split("/") - mydirlist=listdir(self.root+VDB_PATH+"/"+mysplit[0],EmptyOnError=1) - for x in mydirlist: - mypsplit=pkgsplit(x) - if not mypsplit: - self.dbapi.invalidentry(self.root+VDB_PATH+"/"+mysplit[0]+"/"+x) - continue - if mypsplit[0]==mysplit[1]: - return 1 - return 0 - - def populate(self): - self.populated=1 auxdbkeys=[ 'DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', @@ -5613,2264 +4572,11 @@ auxdbkeys=[ ] auxdbkeylen=len(auxdbkeys) -def close_portdbapi_caches(): - for i in portdbapi.portdbapi_instances: - i.close_caches() - - -class portdbapi(dbapi): - """this tree will scan a portage directory located at root (passed to init)""" - portdbapi_instances = [] - - def __init__(self,porttree_root,mysettings=None): - portdbapi.portdbapi_instances.append(self) - - if mysettings: - self.mysettings = mysettings - else: - global settings - self.mysettings = config(clone=settings) - - # This is strictly for use in aux_get() doebuild calls when metadata - # is generated by the depend phase. It's safest to use a clone for - # this purpose because doebuild makes many changes to the config - # instance that is passed in. - self.doebuild_settings = config(clone=self.mysettings) - - self.manifestVerifyLevel = None - self.manifestVerifier = None - self.manifestCache = {} # {location: [stat, md5]} - self.manifestMissingCache = [] - - if "gpg" in self.mysettings.features: - self.manifestVerifyLevel = portage.gpg.EXISTS - if "strict" in self.mysettings.features: - self.manifestVerifyLevel = portage.gpg.MARGINAL - self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel) - elif "severe" in self.mysettings.features: - self.manifestVerifyLevel = portage.gpg.TRUSTED - self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", requireSignedRing=True, minimumTrust=self.manifestVerifyLevel) - else: - self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel) - - #self.root=settings["PORTDIR"] - self.porttree_root = os.path.realpath(porttree_root) - - self.depcachedir = self.mysettings.depcachedir[:] - - self.tmpfs = self.mysettings["PORTAGE_TMPFS"] - if self.tmpfs and not os.path.exists(self.tmpfs): - self.tmpfs = None - if self.tmpfs and not os.access(self.tmpfs, os.W_OK): - self.tmpfs = None - if self.tmpfs and not os.access(self.tmpfs, os.R_OK): - self.tmpfs = None - - self.eclassdb = portage.eclass_cache.cache(self.porttree_root, - overlays=self.mysettings["PORTDIR_OVERLAY"].split()) - - self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule") - - #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening) - self.xcache={} - self.frozen=0 - - self.porttrees = [self.porttree_root] + \ - [os.path.realpath(t) for t in self.mysettings["PORTDIR_OVERLAY"].split()] - self.treemap = {} - for path in self.porttrees: - repo_name_path = os.path.join( path, REPO_NAME_LOC ) - try: - repo_name = open( repo_name_path ,'r').readline().strip() - self.treemap[repo_name] = path - except (OSError,IOError): - pass - - self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule") - self.auxdb = {} - self._init_cache_dirs() - # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys - # ~harring - filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys) - if secpass < 1: - from portage.cache import metadata_overlay, volatile - for x in self.porttrees: - db_ro = self.auxdbmodule(self.depcachedir, x, - filtered_auxdbkeys, gid=portage_gid, readonly=True) - self.auxdb[x] = metadata_overlay.database( - self.depcachedir, x, filtered_auxdbkeys, - gid=portage_gid, db_rw=volatile.database, - db_ro=db_ro) - else: - for x in self.porttrees: - # location, label, auxdbkeys - self.auxdb[x] = self.auxdbmodule( - self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid) - # Selectively cache metadata in order to optimize dep matching. - self._aux_cache_keys = set(["EAPI", "KEYWORDS", "SLOT"]) - self._aux_cache = {} - - def _init_cache_dirs(self): - """Create /var/cache/edb/dep and adjust permissions for the portage - group.""" - - dirmode = 02070 - filemode = 060 - modemask = 02 - - try: - for mydir in (self.depcachedir,): - if portage.util.ensure_dirs(mydir, gid=portage_gid, mode=dirmode, mask=modemask): - writemsg("Adjusting permissions recursively: '%s'\n" % mydir, - noiselevel=-1) - def onerror(e): - raise # bail out on the first error that occurs during recursion - if not apply_recursive_permissions(mydir, - gid=portage_gid, dirmode=dirmode, dirmask=modemask, - filemode=filemode, filemask=modemask, onerror=onerror): - raise portage.exception.OperationNotPermitted( - "Failed to apply recursive permissions for the portage group.") - except portage.exception.PortageException, e: - pass - - def close_caches(self): - for x in self.auxdb.keys(): - self.auxdb[x].sync() - self.auxdb.clear() - - def flush_cache(self): - for x in self.auxdb.values(): - x.sync() - - def finddigest(self,mycpv): - try: - mydig = self.findname2(mycpv)[0] - if not mydig: - return "" - mydigs = mydig.split("/")[:-1] - mydig = "/".join(mydigs) - mysplit = mycpv.split("/") - except OSError: - return "" - return mydig+"/files/digest-"+mysplit[-1] - - def findname(self,mycpv): - return self.findname2(mycpv)[0] - - def getRepositoryPath( self, repository_id ): - """ - This function is required for GLEP 42 compliance; given a valid repository ID - it must return a path to the repository - TreeMap = { id:path } - """ - if repository_id in self.treemap: - return self.treemap[repository_id] - return None - - def getRepositories( self ): - """ - This function is required for GLEP 42 compliance; it will return a list of - repository ID's - TreeMap = { id:path } - """ - return [k for k in self.treemap.keys() if k] - - def findname2(self, mycpv, mytree=None): - """ - Returns the location of the CPV, and what overlay it was in. - Searches overlays first, then PORTDIR; this allows us to return the first - matching file. As opposed to starting in portdir and then doing overlays - second, we would have to exhaustively search the overlays until we found - the file we wanted. - """ - if not mycpv: - return "",0 - mysplit=mycpv.split("/") - psplit=pkgsplit(mysplit[1]) - - if mytree: - mytrees = [mytree] - else: - mytrees = self.porttrees[:] - mytrees.reverse() - if psplit: - for x in mytrees: - file=x+"/"+mysplit[0]+"/"+psplit[0]+"/"+mysplit[1]+".ebuild" - if os.access(file, os.R_OK): - return[file, x] - return None, 0 - - def aux_get(self, mycpv, mylist, mytree=None): - "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." - 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' - 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error' - cache_me = False - if not mytree and not set(mylist).difference(self._aux_cache_keys): - aux_cache = self._aux_cache.get(mycpv) - if aux_cache is not None: - return [aux_cache[x] for x in mylist] - cache_me = True - global auxdbkeys,auxdbkeylen - cat,pkg = mycpv.split("/", 1) - - myebuild, mylocation = self.findname2(mycpv, mytree) - - if not myebuild: - writemsg("!!! aux_get(): ebuild path for '%(cpv)s' not specified:\n" % {"cpv":mycpv}, - noiselevel=1) - writemsg("!!! %s\n" % myebuild, noiselevel=1) - raise KeyError(mycpv) - - myManifestPath = "/".join(myebuild.split("/")[:-1])+"/Manifest" - if "gpg" in self.mysettings.features: - try: - mys = portage.gpg.fileStats(myManifestPath) - if (myManifestPath in self.manifestCache) and \ - (self.manifestCache[myManifestPath] == mys): - pass - elif self.manifestVerifier: - if not self.manifestVerifier.verify(myManifestPath): - # Verification failed the desired level. - raise portage.exception.UntrustedSignature, "Untrusted Manifest: %(manifest)s" % {"manifest":myManifestPath} - - if ("severe" in self.mysettings.features) and \ - (mys != portage.gpg.fileStats(myManifestPath)): - raise portage.exception.SecurityViolation, "Manifest changed: %(manifest)s" % {"manifest":myManifestPath} - - except portage.exception.InvalidSignature, e: - if ("strict" in self.mysettings.features) or \ - ("severe" in self.mysettings.features): - raise - writemsg("!!! INVALID MANIFEST SIGNATURE DETECTED: %(manifest)s\n" % {"manifest":myManifestPath}) - except portage.exception.MissingSignature, e: - if ("severe" in self.mysettings.features): - raise - if ("strict" in self.mysettings.features): - if myManifestPath not in self.manifestMissingCache: - writemsg("!!! WARNING: Missing signature in: %(manifest)s\n" % {"manifest":myManifestPath}) - self.manifestMissingCache.insert(0,myManifestPath) - except (OSError,portage.exception.FileNotFound), e: - if ("strict" in self.mysettings.features) or \ - ("severe" in self.mysettings.features): - raise portage.exception.SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)} - writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath}, - noiselevel=-1) - - - if os.access(myebuild, os.R_OK): - emtime=os.stat(myebuild)[stat.ST_MTIME] - else: - writemsg("!!! aux_get(): ebuild for '%(cpv)s' does not exist at:\n" % {"cpv":mycpv}, - noiselevel=-1) - writemsg("!!! %s\n" % myebuild, - noiselevel=-1) - raise KeyError - - try: - mydata = self.auxdb[mylocation][mycpv] - if emtime != long(mydata.get("_mtime_", 0)): - doregen = True - elif len(mydata.get("_eclasses_", [])) > 0: - doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"]) - else: - doregen = False - - except KeyError: - doregen = True - except CacheError: - doregen = True - try: del self.auxdb[mylocation][mycpv] - except KeyError: pass - - writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2) - - if doregen: - writemsg("doregen: %s %s\n" % (doregen,mycpv), 2) - writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n",1) - - self.doebuild_settings.reset() - mydata = {} - myret = doebuild(myebuild, "depend", - self.doebuild_settings["ROOT"], self.doebuild_settings, - dbkey=mydata, tree="porttree", mydbapi=self) - if myret != os.EX_OK: - raise KeyError(mycpv) - - if "EAPI" not in mydata or not mydata["EAPI"].strip(): - mydata["EAPI"] = "0" - - if not eapi_is_supported(mydata["EAPI"]): - # if newer version, wipe everything and negate eapi - eapi = mydata["EAPI"] - mydata = {} - map(lambda x:mydata.setdefault(x, ""), auxdbkeys) - mydata["EAPI"] = "-"+eapi - - if mydata.get("INHERITED", False): - mydata["_eclasses_"] = self.eclassdb.get_eclass_data(mydata["INHERITED"].split()) - else: - mydata["_eclasses_"] = {} - - del mydata["INHERITED"] - - mydata["_mtime_"] = emtime - - self.auxdb[mylocation][mycpv] = mydata - - if not mydata.setdefault("EAPI", "0"): - mydata["EAPI"] = "0" - - #finally, we look at our internal cache entry and return the requested data. - returnme = [] - for x in mylist: - if x == "INHERITED": - returnme.append(' '.join(mydata.get("_eclasses_", {}).keys())) - else: - returnme.append(mydata.get(x,"")) - - if cache_me: - aux_cache = {} - for x in self._aux_cache_keys: - aux_cache[x] = mydata.get(x, "") - self._aux_cache[mycpv] = aux_cache - - return returnme - - def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None): - if mysettings is None: - mysettings = self.mysettings - try: - myuris = self.aux_get(mypkg, ["SRC_URI"], mytree=mytree)[0] - except KeyError: - print red("getfetchlist():")+" aux_get() error reading "+mypkg+"; aborting." - sys.exit(1) - - if useflags is None: - useflags = mysettings["USE"].split() - - myurilist = portage.dep.paren_reduce(myuris) - myurilist = portage.dep.use_reduce(myurilist,uselist=useflags,matchall=all) - newuris = flatten(myurilist) - - myfiles = [] - for x in newuris: - mya = os.path.basename(x) - if not mya in myfiles: - myfiles.append(mya) - return [newuris, myfiles] - - def getfetchsizes(self,mypkg,useflags=None,debug=0): - # returns a filename:size dictionnary of remaining downloads - myebuild = self.findname(mypkg) - pkgdir = os.path.dirname(myebuild) - mf = Manifest(pkgdir, self.mysettings["DISTDIR"]) - checksums = mf.getDigests() - if not checksums: - if debug: print "[empty/missing/bad digest]: "+mypkg - return None - filesdict={} - if useflags is None: - myuris, myfiles = self.getfetchlist(mypkg,all=1) - else: - myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags) - #XXX: maybe this should be improved: take partial downloads - # into account? check checksums? - for myfile in myfiles: - if myfile not in checksums: - if debug: - writemsg("[bad digest]: missing %s for %s\n" % (myfile, mypkg)) - continue - file_path = os.path.join(self.mysettings["DISTDIR"], myfile) - mystat = None - try: - mystat = os.stat(file_path) - except OSError, e: - pass - if mystat is None: - existing_size = 0 - else: - existing_size = mystat.st_size - remaining_size = int(checksums[myfile]["size"]) - existing_size - if remaining_size > 0: - # Assume the download is resumable. - filesdict[myfile] = remaining_size - elif remaining_size < 0: - # The existing file is too large and therefore corrupt. - filesdict[myfile] = int(checksums[myfile]["size"]) - return filesdict - - def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False): - if not useflags: - if mysettings: - useflags = mysettings["USE"].split() - myuri, myfiles = self.getfetchlist(mypkg, useflags=useflags, mysettings=mysettings, all=all) - myebuild = self.findname(mypkg) - pkgdir = os.path.dirname(myebuild) - mf = Manifest(pkgdir, self.mysettings["DISTDIR"]) - mysums = mf.getDigests() - - failures = {} - for x in myfiles: - if not mysums or x not in mysums: - ok = False - reason = "digest missing" - else: - try: - ok, reason = portage.checksum.verify_all( - os.path.join(self.mysettings["DISTDIR"], x), mysums[x]) - except portage.exception.FileNotFound, e: - ok = False - reason = "File Not Found: '%s'" % str(e) - if not ok: - failures[x] = reason - if failures: - return False - return True - - def getsize(self,mypkg,useflags=None,debug=0): - # returns the total size of remaining downloads - # - # we use getfetchsizes() now, so this function would be obsoleted - # - filesdict=self.getfetchsizes(mypkg,useflags=useflags,debug=debug) - if filesdict is None: - return "[empty/missing/bad digest]" - mysize=0 - for myfile in filesdict.keys(): - mysum+=filesdict[myfile] - return mysum - - def cpv_exists(self,mykey): - "Tells us whether an actual ebuild exists on disk (no masking)" - cps2=mykey.split("/") - cps=catpkgsplit(mykey,silent=0) - if not cps: - #invalid cat/pkg-v - return 0 - if self.findname(cps[0]+"/"+cps2[1]): - return 1 - else: - return 0 - - def cp_all(self): - "returns a list of all keys in our tree" - d={} - for x in self.mysettings.categories: - for oroot in self.porttrees: - for y in listdir(oroot+"/"+x,EmptyOnError=1,ignorecvs=1,dirsonly=1): - d[x+"/"+y] = None - l = d.keys() - l.sort() - return l - - def p_list(self,mycp): - d={} - for oroot in self.porttrees: - for x in listdir(oroot+"/"+mycp,EmptyOnError=1,ignorecvs=1): - if x[-7:]==".ebuild": - d[x[:-7]] = None - return d.keys() - - def cp_list(self, mycp, use_cache=1, mytree=None): - mysplit=mycp.split("/") - d={} - if mytree: - mytrees = [mytree] - else: - mytrees = self.porttrees - for oroot in mytrees: - for x in listdir(oroot+"/"+mycp,EmptyOnError=1,ignorecvs=1): - if x.endswith(".ebuild"): - pf = x[:-7] - ps = pkgsplit(pf) - if not ps: - writemsg("\nInvalid ebuild name: %s\n" % \ - os.path.join(oroot, mycp, x), noiselevel=-1) - continue - d[mysplit[0]+"/"+pf] = None - return d.keys() - - def freeze(self): - for x in ["list-visible","bestmatch-visible","match-visible","match-all"]: - self.xcache[x]={} - self.frozen=1 - - def melt(self): - self.xcache={} - self.frozen=0 - - def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None): - "caching match function; very trick stuff" - #if no updates are being made to the tree, we can consult our xcache... - if self.frozen: - try: - return self.xcache[level][origdep][:] - except KeyError: - pass - - if not mydep: - #this stuff only runs on first call of xmatch() - #create mydep, mykey from origdep - mydep = dep_expand(origdep, mydb=self, settings=self.mysettings) - mykey=dep_getkey(mydep) - - if level=="list-visible": - #a list of all visible packages, not called directly (just by xmatch()) - #myval=self.visible(self.cp_list(mykey)) - myval=self.gvisible(self.visible(self.cp_list(mykey))) - elif level=="bestmatch-visible": - #dep match -- best match of all visible packages - myval=best(self.xmatch("match-visible",None,mydep=mydep,mykey=mykey)) - #get all visible matches (from xmatch()), then choose the best one - elif level=="bestmatch-list": - #dep match -- find best match but restrict search to sublist - myval=best(match_from_list(mydep,mylist)) - #no point is calling xmatch again since we're not caching list deps - elif level=="match-list": - #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible()) - myval=match_from_list(mydep,mylist) - elif level=="match-visible": - #dep match -- find all visible matches - myval = match_from_list(mydep, - self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey)) - #get all visible packages, then get the matching ones - elif level=="match-all": - #match *all* visible *and* masked packages - myval=match_from_list(mydep,self.cp_list(mykey)) - else: - print "ERROR: xmatch doesn't handle",level,"query!" - raise KeyError - myslot = portage.dep.dep_getslot(mydep) - if myslot is not None: - slotmatches = [] - for cpv in myval: - try: - if self.aux_get(cpv, ["SLOT"])[0] == myslot: - slotmatches.append(cpv) - except KeyError: - pass # ebuild masked by corruption - myval = slotmatches - if self.frozen and (level not in ["match-list","bestmatch-list"]): - self.xcache[level][mydep]=myval - if origdep and origdep != mydep: - self.xcache[level][origdep] = myval - return myval[:] - - def match(self,mydep,use_cache=1): - return self.xmatch("match-visible",mydep) - - def visible(self,mylist): - """two functions in one. Accepts a list of cpv values and uses the package.mask *and* - packages file to remove invisible entries, returning remaining items. This function assumes - that all entries in mylist have the same category and package name.""" - if (mylist is None) or (len(mylist)==0): - return [] - newlist=mylist[:] - #first, we mask out packages in the package.mask file - mykey=newlist[0] - cpv=catpkgsplit(mykey) - if not cpv: - #invalid cat/pkg-v - print "visible(): invalid cat/pkg-v:",mykey - return [] - mycp=cpv[0]+"/"+cpv[1] - maskdict=self.mysettings.pmaskdict - unmaskdict=self.mysettings.punmaskdict - if maskdict.has_key(mycp): - for x in maskdict[mycp]: - mymatches=self.xmatch("match-all",x) - if mymatches is None: - #error in package.mask file; print warning and continue: - print "visible(): package.mask entry \""+x+"\" is invalid, ignoring..." - continue - for y in mymatches: - unmask=0 - if unmaskdict.has_key(mycp): - for z in unmaskdict[mycp]: - mymatches_unmask=self.xmatch("match-all",z) - if y in mymatches_unmask: - unmask=1 - break - if unmask==0: - try: - newlist.remove(y) - except ValueError: - pass - - revmaskdict=self.mysettings.prevmaskdict - if revmaskdict.has_key(mycp): - for x in revmaskdict[mycp]: - #important: only match against the still-unmasked entries... - #notice how we pass "newlist" to the xmatch() call below.... - #Without this, ~ deps in the packages files are broken. - mymatches=self.xmatch("match-list",x,mylist=newlist) - if mymatches is None: - #error in packages file; print warning and continue: - print "emerge: visible(): profile packages entry \""+x+"\" is invalid, ignoring..." - continue - pos=0 - while pos "+mynewcpv+"\n", noiselevel=-1) - continue - - tbz2path=self.getname(mycpv) - if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK): - writemsg("!!! Cannot update readonly binary: "+mycpv+"\n", - noiselevel=-1) - continue - - #print ">>> Updating data in:",mycpv - writemsg_stdout("%") - mytbz2 = portage.xpak.tbz2(tbz2path) - mydata = mytbz2.get_data() - updated_items = update_dbentries([mylist], mydata) - mydata.update(updated_items) - mydata["CATEGORY"] = mynewcat+"\n" - if mynewpkg != myoldpkg: - mydata[mynewpkg+".ebuild"] = mydata[myoldpkg+".ebuild"] - del mydata[myoldpkg+".ebuild"] - mydata["PF"] = mynewpkg + "\n" - mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) - - self.dbapi.cpv_remove(mycpv) - del self._pkg_paths[mycpv] - new_path = self.getname(mynewcpv) - self._pkg_paths[mynewcpv] = os.path.join( - *new_path.split(os.path.sep)[-2:]) - if new_path != mytbz2: - try: - os.makedirs(os.path.dirname(new_path)) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e - os.rename(tbz2path, new_path) - self._remove_symlink(mycpv) - if new_path.split(os.path.sep)[-2] == "All": - self._create_symlink(mynewcpv) - self.dbapi.cpv_inject(mynewcpv) - - return 1 - - def _remove_symlink(self, cpv): - """Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also remove - the ${PKGDIR}/${CATEGORY} directory if empty. The file will not be - removed if os.path.islink() returns False.""" - mycat, mypkg = catsplit(cpv) - mylink = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2") - if os.path.islink(mylink): - """Only remove it if it's really a link so that this method never - removes a real package that was placed here to avoid a collision.""" - os.unlink(mylink) - try: - os.rmdir(os.path.join(self.pkgdir, mycat)) - except OSError, e: - if e.errno not in (errno.ENOENT, - errno.ENOTEMPTY, errno.EEXIST): - raise - del e - - def _create_symlink(self, cpv): - """Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and - ${PKGDIR}/${CATEGORY} directory, if necessary). Any file that may - exist in the location of the symlink will first be removed.""" - mycat, mypkg = catsplit(cpv) - full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2") - try: - os.makedirs(os.path.dirname(full_path)) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e - try: - os.unlink(full_path) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e - os.symlink(os.path.join("..", "All", mypkg + ".tbz2"), full_path) - - def move_slot_ent(self, mylist): - if not self.populated: - self.populate() - pkg=mylist[1] - origslot=mylist[2] - newslot=mylist[3] - - if not isvalidatom(pkg): - raise portage.exception.InvalidAtom(pkg) - - origmatches=self.dbapi.match(pkg) - if not origmatches: - return - for mycpv in origmatches: - mycpsplit=catpkgsplit(mycpv) - myoldpkg=mycpv.split("/")[1] - tbz2path=self.getname(mycpv) - if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK): - writemsg("!!! Cannot update readonly binary: "+mycpv+"\n", - noiselevel=-1) - continue - - #print ">>> Updating data in:",mycpv - mytbz2 = portage.xpak.tbz2(tbz2path) - mydata = mytbz2.get_data() - - slot = mydata["SLOT"] - if (not slot): - continue - - if (slot[0]!=origslot): - continue - - writemsg_stdout("S") - mydata["SLOT"] = newslot+"\n" - mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) - return 1 - - def update_ents(self, update_iter): - if len(update_iter) == 0: - return - if not self.populated: - self.populate() - - for mycpv in self.dbapi.cp_all(): - tbz2path=self.getname(mycpv) - if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK): - writemsg("!!! Cannot update readonly binary: "+mycpv+"\n", - noiselevel=-1) - continue - #print ">>> Updating binary data:",mycpv - writemsg_stdout("*") - mytbz2 = portage.xpak.tbz2(tbz2path) - mydata = mytbz2.get_data() - updated_items = update_dbentries(update_iter, mydata) - if len(updated_items) > 0: - mydata.update(updated_items) - mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) - return 1 - - def prevent_collision(self, cpv): - """Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is safe to - use for a given cpv. If a collision will occur with an existing - package from another category, the existing package will be bumped to - ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist.""" - full_path = self.getname(cpv) - if "All" == full_path.split(os.path.sep)[-2]: - return - """Move a colliding package if it exists. Code below this point only - executes in rare cases.""" - mycat, mypkg = catsplit(cpv) - myfile = mypkg + ".tbz2" - mypath = os.path.join("All", myfile) - dest_path = os.path.join(self.pkgdir, mypath) - if os.path.exists(dest_path): - # For invalid packages, other_cat could be None. - other_cat = portage.xpak.tbz2(dest_path).getfile("CATEGORY") - if other_cat: - other_cat = other_cat.strip() - self._move_from_all(other_cat + "/" + mypkg) - """The file may or may not exist. Move it if necessary and update - internal state for future calls to getname().""" - self._move_to_all(cpv) - - def _move_to_all(self, cpv): - """If the file exists, move it. Whether or not it exists, update state - for future getname() calls.""" - mycat , mypkg = catsplit(cpv) - myfile = mypkg + ".tbz2" - src_path = os.path.join(self.pkgdir, mycat, myfile) - try: - mystat = os.lstat(src_path) - except OSError, e: - mystat = None - if mystat and stat.S_ISREG(mystat.st_mode): - try: - os.makedirs(os.path.join(self.pkgdir, "All")) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e - os.rename(src_path, os.path.join(self.pkgdir, "All", myfile)) - self._create_symlink(cpv) - self._pkg_paths[cpv] = os.path.join("All", myfile) - - def _move_from_all(self, cpv): - """Move a package from ${PKGDIR}/All/${PF}.tbz2 to - ${PKGDIR}/${CATEGORY}/${PF}.tbz2 and update state from getname calls.""" - self._remove_symlink(cpv) - mycat , mypkg = catsplit(cpv) - myfile = mypkg + ".tbz2" - mypath = os.path.join(mycat, myfile) - dest_path = os.path.join(self.pkgdir, mypath) - try: - os.makedirs(os.path.dirname(dest_path)) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e - os.rename(os.path.join(self.pkgdir, "All", myfile), dest_path) - self._pkg_paths[cpv] = mypath - - def populate(self, getbinpkgs=0,getbinpkgsonly=0): - "populates the binarytree" - if (not os.path.isdir(self.pkgdir) and not getbinpkgs): - return 0 - if (not os.path.isdir(self.pkgdir+"/All") and not getbinpkgs): - return 0 - - if not getbinpkgsonly: - pkg_paths = {} - dirs = listdir(self.pkgdir, dirsonly=True, EmptyOnError=True) - if "All" in dirs: - dirs.remove("All") - dirs.sort() - dirs.insert(0, "All") - for mydir in dirs: - for myfile in listdir(os.path.join(self.pkgdir, mydir)): - if not myfile.endswith(".tbz2"): - continue - mypath = os.path.join(mydir, myfile) - full_path = os.path.join(self.pkgdir, mypath) - if os.path.islink(full_path): - continue - mytbz2 = portage.xpak.tbz2(full_path) - # For invalid packages, mycat could be None. - mycat = mytbz2.getfile("CATEGORY") - mypf = mytbz2.getfile("PF") - mypkg = myfile[:-5] - if not mycat or not mypf: - #old-style or corrupt package - writemsg("!!! Invalid binary package: '%s'\n" % full_path, - noiselevel=-1) - writemsg("!!! This binary package is not " + \ - "recoverable and should be deleted.\n", - noiselevel=-1) - self.invalids.append(mypkg) - continue - mycat = mycat.strip() - if mycat != mydir and mydir != "All": - continue - if mypkg != mypf.strip(): - continue - mycpv = mycat + "/" + mypkg - if mycpv in pkg_paths: - # All is first, so it's preferred. - continue - pkg_paths[mycpv] = mypath - self.dbapi.cpv_inject(mycpv) - self._pkg_paths = pkg_paths - - if getbinpkgs and not self.settings["PORTAGE_BINHOST"]: - writemsg(red("!!! PORTAGE_BINHOST unset, but use is requested.\n"), - noiselevel=-1) - - if getbinpkgs and \ - self.settings["PORTAGE_BINHOST"] and not self.remotepkgs: - try: - chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"]) - if chunk_size < 8: - chunk_size = 8 - except (ValueError, KeyError): - chunk_size = 3000 - - writemsg(green("Fetching binary packages info...\n")) - self.remotepkgs = portage.getbinpkg.dir_get_metadata( - self.settings["PORTAGE_BINHOST"], chunk_size=chunk_size) - writemsg(green(" -- DONE!\n\n")) - - for mypkg in self.remotepkgs.keys(): - if not self.remotepkgs[mypkg].has_key("CATEGORY"): - #old-style or corrupt package - writemsg("!!! Invalid remote binary package: "+mypkg+"\n", - noiselevel=-1) - del self.remotepkgs[mypkg] - continue - mycat=self.remotepkgs[mypkg]["CATEGORY"].strip() - fullpkg=mycat+"/"+mypkg[:-5] - mykey=dep_getkey(fullpkg) - try: - # invalid tbz2's can hurt things. - #print "cpv_inject("+str(fullpkg)+")" - self.dbapi.cpv_inject(fullpkg) - #print " -- Injected" - except SystemExit, e: - raise - except: - writemsg("!!! Failed to inject remote binary package:"+str(fullpkg)+"\n", - noiselevel=-1) - del self.remotepkgs[mypkg] - continue - self.populated=1 - - def inject(self,cpv): - return self.dbapi.cpv_inject(cpv) - - def exists_specific(self,cpv): - if not self.populated: - self.populate() - return self.dbapi.match( - dep_expand("="+cpv, mydb=self.dbapi, settings=self.settings)) - - def dep_bestmatch(self,mydep): - "compatibility method -- all matches, not just visible ones" - if not self.populated: - self.populate() - writemsg("\n\n", 1) - writemsg("mydep: %s\n" % mydep, 1) - mydep = dep_expand(mydep, mydb=self.dbapi, settings=self.settings) - writemsg("mydep: %s\n" % mydep, 1) - mykey=dep_getkey(mydep) - writemsg("mykey: %s\n" % mykey, 1) - mymatch=best(match_from_list(mydep,self.dbapi.cp_list(mykey))) - writemsg("mymatch: %s\n" % mymatch, 1) - if mymatch is None: - return "" - return mymatch - - def getname(self,pkgname): - """Returns a file location for this package. The default location is - ${PKGDIR}/All/${PF}.tbz2, but will be ${PKGDIR}/${CATEGORY}/${PF}.tbz2 - in the rare event of a collision. The prevent_collision() method can - be called to ensure that ${PKGDIR}/All/${PF}.tbz2 is available for a - specific cpv.""" - if not self.populated: - self.populate() - mycpv = pkgname - mypath = self._pkg_paths.get(mycpv, None) - if mypath: - return os.path.join(self.pkgdir, mypath) - mycat, mypkg = catsplit(mycpv) - mypath = os.path.join("All", mypkg + ".tbz2") - if mypath in self._pkg_paths.values(): - mypath = os.path.join(mycat, mypkg + ".tbz2") - self._pkg_paths[mycpv] = mypath # cache for future lookups - return os.path.join(self.pkgdir, mypath) - - def isremote(self,pkgname): - "Returns true if the package is kept remotely." - mysplit=pkgname.split("/") - remote = (not os.path.exists(self.getname(pkgname))) and self.remotepkgs.has_key(mysplit[1]+".tbz2") - return remote - - def get_use(self,pkgname): - mysplit=pkgname.split("/") - if self.isremote(pkgname): - return self.remotepkgs[mysplit[1]+".tbz2"]["USE"][:].split() - tbz2=portage.xpak.tbz2(self.getname(pkgname)) - return tbz2.getfile("USE").split() - - def gettbz2(self,pkgname): - "fetches the package from a remote site, if necessary." - print "Fetching '"+str(pkgname)+"'" - mysplit = pkgname.split("/") - tbz2name = mysplit[1]+".tbz2" - if not self.isremote(pkgname): - if (tbz2name not in self.invalids): - return - else: - writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n", - noiselevel=-1) - mydest = self.pkgdir+"/All/" - try: - os.makedirs(mydest, 0775) - except (OSError, IOError): - pass - return portage.getbinpkg.file_get( - self.settings["PORTAGE_BINHOST"] + "/" + tbz2name, - mydest, fcmd=self.settings["RESUMECOMMAND"]) - - def getslot(self,mycatpkg): - "Get a slot for a catpkg; assume it exists." - myslot = "" - try: - myslot=self.dbapi.aux_get(mycatpkg,["SLOT"])[0] - except SystemExit, e: - raise - except Exception, e: - pass - return myslot - -class dblink: - """ - This class provides an interface to the installed package database - At present this is implemented as a text backend in /var/db/pkg. - """ - def __init__(self, cat, pkg, myroot, mysettings, treetype=None, - vartree=None): - """ - Creates a DBlink object for a given CPV. - The given CPV may not be present in the database already. - - @param cat: Category - @type cat: String - @param pkg: Package (PV) - @type pkg: String - @param myroot: Typically ${ROOT} - @type myroot: String (Path) - @param mysettings: Typically portage.config - @type mysettings: An instance of portage.config - @param treetype: one of ['porttree','bintree','vartree'] - @type treetype: String - @param vartree: an instance of vartree corresponding to myroot. - @type vartree: vartree - """ - - self.cat = cat - self.pkg = pkg - self.mycpv = self.cat+"/"+self.pkg - self.mysplit = pkgsplit(self.mycpv) - self.treetype = treetype - if vartree is None: - global db - vartree = db[myroot]["vartree"] - self.vartree = vartree - - self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH)) - self.dbcatdir = self.dbroot+"/"+cat - self.dbpkgdir = self.dbcatdir+"/"+pkg - self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg - self.dbdir = self.dbpkgdir - - self._lock_vdb = None - - self.settings = mysettings - if self.settings==1: - raise ValueError - - self.myroot=myroot - protect_obj = portage.util.ConfigProtect(myroot, - mysettings.get("CONFIG_PROTECT","").split(), - mysettings.get("CONFIG_PROTECT_MASK","").split()) - self.updateprotect = protect_obj.updateprotect - self._config_protect = protect_obj - self._installed_instance = None - self.contentscache=[] - self._contents_inodes = None - - def lockdb(self): - if self._lock_vdb: - raise AssertionError("Lock already held.") - # At least the parent needs to exist for the lock file. - portage.util.ensure_dirs(self.dbroot) - self._lock_vdb = portage.locks.lockdir(self.dbroot) - - def unlockdb(self): - if self._lock_vdb: - portage.locks.unlockdir(self._lock_vdb) - self._lock_vdb = None - - def getpath(self): - "return path to location of db information (for >>> informational display)" - return self.dbdir - - def exists(self): - "does the db entry exist? boolean." - return os.path.exists(self.dbdir) - - def create(self): - "create the skeleton db directory structure. No contents, virtuals, provides or anything. Also will create /var/db/pkg if necessary." - """ - This function should never get called (there is no reason to use it). - """ - # XXXXX Delete this eventually - raise Exception, "This is bad. Don't use it." - if not os.path.exists(self.dbdir): - os.makedirs(self.dbdir) - - def delete(self): - """ - Remove this entry from the database - """ - if not os.path.exists(self.dbdir): - return - try: - for x in listdir(self.dbdir): - os.unlink(self.dbdir+"/"+x) - os.rmdir(self.dbdir) - except OSError, e: - print "!!! Unable to remove db entry for this package." - print "!!! It is possible that a directory is in this one. Portage will still" - print "!!! register this package as installed as long as this directory exists." - print "!!! You may delete this directory with 'rm -Rf "+self.dbdir+"'" - print "!!! "+str(e) - print - sys.exit(1) - - def clearcontents(self): - """ - For a given db entry (self), erase the CONTENTS values. - """ - if os.path.exists(self.dbdir+"/CONTENTS"): - os.unlink(self.dbdir+"/CONTENTS") - - def getcontents(self): - """ - Get the installed files of a given package (aka what that package installed) - """ - if not os.path.exists(self.dbdir+"/CONTENTS"): - return None - if self.contentscache != []: - return self.contentscache - pkgfiles={} - myc=open(self.dbdir+"/CONTENTS","r") - mylines=myc.readlines() - myc.close() - null_byte = "\0" - contents_file = os.path.join(self.dbdir, "CONTENTS") - pos = 0 - for line in mylines: - pos += 1 - if null_byte in line: - # Null bytes are a common indication of corruption. - writemsg("!!! Null byte found in contents " + \ - "file, line %d: '%s'\n" % (pos, contents_file), - noiselevel=-1) - continue - mydat = line.split() - # we do this so we can remove from non-root filesystems - # (use the ROOT var to allow maintenance on other partitions) - try: - mydat[1] = normalize_path(os.path.join( - self.myroot, mydat[1].lstrip(os.path.sep))) - if mydat[0]=="obj": - #format: type, mtime, md5sum - pkgfiles[" ".join(mydat[1:-2])]=[mydat[0], mydat[-1], mydat[-2]] - elif mydat[0]=="dir": - #format: type - pkgfiles[" ".join(mydat[1:])]=[mydat[0] ] - elif mydat[0]=="sym": - #format: type, mtime, dest - x=len(mydat)-1 - if (x >= 13) and (mydat[-1][-1]==')'): # Old/Broken symlink entry - mydat = mydat[:-10]+[mydat[-10:][stat.ST_MTIME][:-1]] - writemsg("FIXED SYMLINK LINE: %s\n" % mydat, 1) - x=len(mydat)-1 - splitter=-1 - while(x>=0): - if mydat[x]=="->": - splitter=x - break - x=x-1 - if splitter==-1: - return None - pkgfiles[" ".join(mydat[1:splitter])]=[mydat[0], mydat[-1], " ".join(mydat[(splitter+1):-1])] - elif mydat[0]=="dev": - #format: type - pkgfiles[" ".join(mydat[1:])]=[mydat[0] ] - elif mydat[0]=="fif": - #format: type - pkgfiles[" ".join(mydat[1:])]=[mydat[0]] - else: - return None - except (KeyError,IndexError): - print "portage: CONTENTS line",pos,"corrupt!" - self.contentscache=pkgfiles - return pkgfiles - - def unmerge(self, pkgfiles=None, trimworld=1, cleanup=1, - ldpath_mtimes=None): - """ - Calls prerm - Unmerges a given package (CPV) - calls postrm - calls cleanrm - calls env_update - - @param pkgfiles: files to unmerge (generally self.getcontents() ) - @type pkgfiles: Dictionary - @param trimworld: Remove CPV from world file if True, not if False - @type trimworld: Boolean - @param cleanup: cleanup to pass to doebuild (see doebuild) - @type cleanup: Boolean - @param ldpath_mtimes: mtimes to pass to env_update (see env_update) - @type ldpath_mtimes: Dictionary - @rtype: Integer - @returns: - 1. os.EX_OK if everything went well. - 2. return code of the failed phase (for prerm, postrm, cleanrm) - - Notes: - The caller must ensure that lockdb() and unlockdb() are called - before and after this method. - """ - - contents = self.getcontents() - # Now, don't assume that the name of the ebuild is the same as the - # name of the dir; the package may have been moved. - myebuildpath = None - mystuff = listdir(self.dbdir, EmptyOnError=1) - for x in mystuff: - if x.endswith(".ebuild"): - myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild") - if x[:-7] != self.pkg: - # Clean up after vardbapi.move_ent() breakage in - # portage versions before 2.1.2 - os.rename(os.path.join(self.dbdir, x), myebuildpath) - write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n") - break - - self.settings.load_infodir(self.dbdir) - if myebuildpath: - try: - doebuild_environment(myebuildpath, "prerm", self.myroot, - self.settings, 0, 0, self.vartree.dbapi) - except portage.exception.UnsupportedAPIException, e: - # Sometimes this happens due to corruption of the EAPI file. - writemsg("!!! FAILED prerm: %s\n" % \ - os.path.join(self.dbdir, "EAPI"), noiselevel=-1) - writemsg("%s\n" % str(e), noiselevel=-1) - return 1 - catdir = os.path.dirname(self.settings["PORTAGE_BUILDDIR"]) - portage.util.ensure_dirs(os.path.dirname(catdir), - uid=portage_uid, gid=portage_gid, mode=070, mask=0) - builddir_lock = None - catdir_lock = None - try: - if myebuildpath: - catdir_lock = portage.locks.lockdir(catdir) - portage.util.ensure_dirs(catdir, - uid=portage_uid, gid=portage_gid, - mode=070, mask=0) - builddir_lock = portage.locks.lockdir( - self.settings["PORTAGE_BUILDDIR"]) - try: - portage.locks.unlockdir(catdir_lock) - finally: - catdir_lock = None - # Eventually, we'd like to pass in the saved ebuild env here... - retval = doebuild(myebuildpath, "prerm", self.myroot, - self.settings, cleanup=cleanup, use_cache=0, - mydbapi=self.vartree.dbapi, tree="vartree", - vartree=self.vartree) - # XXX: Decide how to handle failures here. - if retval != os.EX_OK: - writemsg("!!! FAILED prerm: %s\n" % retval, noiselevel=-1) - return retval - - self._unmerge_pkgfiles(pkgfiles) - - if myebuildpath: - retval = doebuild(myebuildpath, "postrm", self.myroot, - self.settings, use_cache=0, tree="vartree", - mydbapi=self.vartree.dbapi, vartree=self.vartree) - - # process logs created during pre/postrm - elog_process(self.mycpv, self.settings) - - # XXX: Decide how to handle failures here. - if retval != os.EX_OK: - writemsg("!!! FAILED postrm: %s\n" % retval, noiselevel=-1) - return retval - doebuild(myebuildpath, "cleanrm", self.myroot, self.settings, - tree="vartree", mydbapi=self.vartree.dbapi, - vartree=self.vartree) - - finally: - if builddir_lock: - portage.locks.unlockdir(builddir_lock) - try: - if myebuildpath and not catdir_lock: - # Lock catdir for removal if empty. - catdir_lock = portage.locks.lockdir(catdir) - finally: - if catdir_lock: - try: - os.rmdir(catdir) - except OSError, e: - if e.errno not in (errno.ENOENT, - errno.ENOTEMPTY, errno.EEXIST): - raise - del e - portage.locks.unlockdir(catdir_lock) - env_update(target_root=self.myroot, prev_mtimes=ldpath_mtimes, - contents=contents) - return os.EX_OK - - def _unmerge_pkgfiles(self, pkgfiles): - """ - - Unmerges the contents of a package from the liveFS - Removes the VDB entry for self - - @param pkgfiles: typically self.getcontents() - @type pkgfiles: Dictionary { filename: [ 'type', '?', 'md5sum' ] } - @rtype: None - """ - global dircache - dircache={} - - if not pkgfiles: - writemsg_stdout("No package files given... Grabbing a set.\n") - pkgfiles=self.getcontents() - - if pkgfiles: - mykeys=pkgfiles.keys() - mykeys.sort() - mykeys.reverse() - - #process symlinks second-to-last, directories last. - mydirs=[] - modprotect="/lib/modules/" - for objkey in mykeys: - obj = normalize_path(objkey) - if obj[:2]=="//": - obj=obj[1:] - statobj = None - try: - statobj = os.stat(obj) - except OSError: - pass - lstatobj = None - try: - lstatobj = os.lstat(obj) - except (OSError, AttributeError): - pass - islink = lstatobj is not None and stat.S_ISLNK(lstatobj.st_mode) - if statobj is None: - if not islink: - #we skip this if we're dealing with a symlink - #because os.stat() will operate on the - #link target rather than the link itself. - writemsg_stdout("--- !found "+str(pkgfiles[objkey][0])+ " %s\n" % obj) - continue - # next line includes a tweak to protect modules from being unmerged, - # but we don't protect modules from being overwritten if they are - # upgraded. We effectively only want one half of the config protection - # functionality for /lib/modules. For portage-ng both capabilities - # should be able to be independently specified. - if obj.startswith(modprotect): - writemsg_stdout("--- cfgpro %s %s\n" % (pkgfiles[objkey][0], obj)) - continue - - lmtime=str(lstatobj[stat.ST_MTIME]) - if (pkgfiles[objkey][0] not in ("dir","fif","dev")) and (lmtime != pkgfiles[objkey][1]): - writemsg_stdout("--- !mtime %s %s\n" % (pkgfiles[objkey][0], obj)) - continue - - if pkgfiles[objkey][0]=="dir": - if statobj is None or not stat.S_ISDIR(statobj.st_mode): - writemsg_stdout("--- !dir %s %s\n" % ("dir", obj)) - continue - mydirs.append(obj) - elif pkgfiles[objkey][0]=="sym": - if not islink: - writemsg_stdout("--- !sym %s %s\n" % ("sym", obj)) - continue - try: - os.unlink(obj) - writemsg_stdout("<<< %s %s\n" % ("sym",obj)) - except (OSError,IOError),e: - writemsg_stdout("!!! %s %s\n" % ("sym",obj)) - elif pkgfiles[objkey][0]=="obj": - if statobj is None or not stat.S_ISREG(statobj.st_mode): - writemsg_stdout("--- !obj %s %s\n" % ("obj", obj)) - continue - mymd5 = None - try: - mymd5 = portage.checksum.perform_md5(obj, calc_prelink=1) - except portage.exception.FileNotFound, e: - # the file has disappeared between now and our stat call - writemsg_stdout("--- !obj %s %s\n" % ("obj", obj)) - continue - - # string.lower is needed because db entries used to be in upper-case. The - # string.lower allows for backwards compatibility. - if mymd5 != pkgfiles[objkey][2].lower(): - writemsg_stdout("--- !md5 %s %s\n" % ("obj", obj)) - continue - try: - os.unlink(obj) - except (OSError,IOError),e: - pass - writemsg_stdout("<<< %s %s\n" % ("obj",obj)) - elif pkgfiles[objkey][0]=="fif": - if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]): - writemsg_stdout("--- !fif %s %s\n" % ("fif", obj)) - continue - writemsg_stdout("--- %s %s\n" % ("fif",obj)) - elif pkgfiles[objkey][0]=="dev": - writemsg_stdout("--- %s %s\n" % ("dev",obj)) - - mydirs.sort() - mydirs.reverse() - - for obj in mydirs: - try: - os.rmdir(obj) - writemsg_stdout("<<< %s %s\n" % ("dir",obj)) - except (OSError, IOError): - writemsg_stdout("--- !empty dir %s\n" % obj) - - #remove self from vartree database so that our own virtual gets zapped if we're the last node - self.vartree.zap(self.mycpv) - - def isowner(self,filename,destroot): - """ - Check if filename is a new file or belongs to this package - (for this or a previous version) - - @param filename: - @type filename: - @param destroot: - @type destroot: - @rtype: Boolean - @returns: - 1. True if this package owns the file. - 2. False if this package does not own the file. - """ - destfile = normalize_path( - os.path.join(destroot, filename.lstrip(os.path.sep))) - try: - mylstat = os.lstat(destfile) - except (OSError, IOError): - return True - - pkgfiles = self.getcontents() - if pkgfiles and filename in pkgfiles: - return True - if pkgfiles: - if self._contents_inodes is None: - self._contents_inodes = set() - for x in pkgfiles: - try: - lstat = os.lstat(x) - self._contents_inodes.add((lstat.st_dev, lstat.st_ino)) - except OSError: - pass - if (mylstat.st_dev, mylstat.st_ino) in self._contents_inodes: - return True - - return False - - def isprotected(self, filename): - """In cases where an installed package in the same slot owns a - protected file that will be merged, bump the mtime on the installed - file in order to ensure that it isn't unmerged.""" - if not self._config_protect.isprotected(filename): - return False - if self._installed_instance is None: - return True - mydata = self._installed_instance.getcontents().get(filename, None) - if mydata is None: - return True - - # Bump the mtime in order to ensure that the old config file doesn't - # get unmerged. The user will have an opportunity to merge the new - # config with the old one. - try: - os.utime(filename, None) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e - # The file has disappeared, so it's not protected. - return False - return True - - def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0, - mydbapi=None, prev_mtimes=None): - """ - - This function does the following: - - Collision Protection. - calls doebuild(mydo=pkg_preinst) - Merges the package to the livefs - unmerges old version (if required) - calls doebuild(mydo=pkg_postinst) - calls env_update - - @param srcroot: Typically this is ${D} - @type srcroot: String (Path) - @param destroot: Path to merge to (usually ${ROOT}) - @type destroot: String (Path) - @param inforoot: root of the vardb entry ? - @type inforoot: String (Path) - @param myebuild: path to the ebuild that we are processing - @type myebuild: String (Path) - @param mydbapi: dbapi which is handed to doebuild. - @type mydbapi: portdbapi instance - @param prev_mtimes: { Filename:mtime } mapping for env_update - @type prev_mtimes: Dictionary - @rtype: Boolean - @returns: - 1. 0 on success - 2. 1 on failure - - secondhand is a list of symlinks that have been skipped due to their target - not existing; we will merge these symlinks at a later time. - """ - if not os.path.isdir(srcroot): - writemsg("!!! Directory Not Found: D='%s'\n" % srcroot, - noiselevel=-1) - return 1 - - if not os.path.exists(self.dbcatdir): - os.makedirs(self.dbcatdir) - - otherversions=[] - for v in self.vartree.dbapi.cp_list(self.mysplit[0]): - otherversions.append(v.split("/")[1]) - - slot_matches = self.vartree.dbapi.match( - "%s:%s" % (self.mysplit[0], self.settings["SLOT"])) - if slot_matches: - # Used by self.isprotected(). - self._installed_instance = dblink(self.cat, - catsplit(slot_matches[0])[1], destroot, self.settings, - vartree=self.vartree) - - # check for package collisions - if "collision-protect" in self.settings.features: - collision_ignore = set([normalize_path(myignore) for myignore in \ - self.settings.get("COLLISION_IGNORE", "").split()]) - myfilelist = listdir(srcroot, recursive=1, filesonly=1, followSymlinks=False) - - # the linkcheck only works if we are in srcroot - mycwd = getcwd() - os.chdir(srcroot) - mysymlinks = filter(os.path.islink, listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False)) - myfilelist.extend(mysymlinks) - mysymlinked_directories = [s + os.path.sep for s in mysymlinks] - del mysymlinks - - - stopmerge=False - starttime=time.time() - i=0 - - otherpkg=[] - mypkglist=[] - - if self.pkg in otherversions: - otherversions.remove(self.pkg) # we already checked this package - - myslot = self.settings["SLOT"] - for v in otherversions: - # only allow versions with same slot to overwrite files - if myslot == self.vartree.dbapi.aux_get("/".join((self.cat, v)), ["SLOT"])[0]: - mypkglist.append( - dblink(self.cat, v, destroot, self.settings, - vartree=self.vartree)) - - collisions = [] - - print green("*")+" checking "+str(len(myfilelist))+" files for package collisions" - for f in myfilelist: - nocheck = False - # listdir isn't intelligent enough to exclude symlinked dirs, - # so we have to do it ourself - for s in mysymlinked_directories: - if f.startswith(s): - nocheck = True - break - if nocheck: - continue - i=i+1 - if i % 1000 == 0: - print str(i)+" files checked ..." - if f[0] != "/": - f="/"+f - isowned = False - for ver in [self]+mypkglist: - if (ver.isowner(f, destroot) or ver.isprotected(f)): - isowned = True - break - if not isowned: - collisions.append(f) - print "existing file "+f+" is not owned by this package" - stopmerge=True - if collision_ignore: - if f in collision_ignore: - stopmerge = False - else: - for myignore in collision_ignore: - if f.startswith(myignore + os.path.sep): - stopmerge = False - break - #print green("*")+" spent "+str(time.time()-starttime)+" seconds checking for file collisions" - if stopmerge: - print red("*")+" This package is blocked because it wants to overwrite" - print red("*")+" files belonging to other packages (see messages above)." - print red("*")+" If you have no clue what this is all about report it " - print red("*")+" as a bug for this package on http://bugs.gentoo.org" - print - print red("package "+self.cat+"/"+self.pkg+" NOT merged") - print - print - print "Searching all installed packages for file collisions..." - print "Press Ctrl-C to Stop" - print - """ Note: The isowner calls result in a stat call for *every* - single installed file, since the inode numbers are used to work - around the problem of ambiguous paths caused by symlinked files - and/or directories. Though it is slow, it is as accurate as - possible.""" - found_owner = False - for cpv in self.vartree.dbapi.cpv_all(): - cat, pkg = catsplit(cpv) - mylink = dblink(cat, pkg, destroot, self.settings, - vartree=self.vartree) - mycollisions = [] - for f in collisions: - if mylink.isowner(f, destroot): - mycollisions.append(f) - if mycollisions: - found_owner = True - print " * %s:" % cpv - print - for f in mycollisions: - print " '%s'" % \ - os.path.join(destroot, f.lstrip(os.path.sep)) - print - if not found_owner: - print "None of the installed packages claim the above file(s)." - print - sys.exit(1) - try: - os.chdir(mycwd) - except OSError: - pass - - if os.stat(srcroot).st_dev == os.stat(destroot).st_dev: - """ The merge process may move files out of the image directory, - which causes invalidation of the .installed flag.""" - try: - os.unlink(os.path.join( - os.path.dirname(normalize_path(srcroot)), ".installed")) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e - - # get old contents info for later unmerging - oldcontents = self.getcontents() - - self.dbdir = self.dbtmpdir - self.delete() - if not os.path.exists(self.dbtmpdir): - os.makedirs(self.dbtmpdir) - - writemsg_stdout(">>> Merging %s %s %s\n" % (self.mycpv,"to",destroot)) - - # run preinst script - if myebuild is None: - myebuild = os.path.join(inforoot, self.pkg + ".ebuild") - a = doebuild(myebuild, "preinst", destroot, self.settings, cleanup=cleanup, - use_cache=0, tree=self.treetype, mydbapi=mydbapi, - vartree=self.vartree) - - # XXX: Decide how to handle failures here. - if a != os.EX_OK: - writemsg("!!! FAILED preinst: "+str(a)+"\n", noiselevel=-1) - return a - - # copy "info" files (like SLOT, CFLAGS, etc.) into the database - for x in listdir(inforoot): - self.copyfile(inforoot+"/"+x) - - # get current counter value (counter_tick also takes care of incrementing it) - # XXX Need to make this destroot, but it needs to be initialized first. XXX - # XXX bis: leads to some invalidentry() call through cp_all(). - counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) - # write local package counter for recording - lcfile = open(self.dbtmpdir+"/COUNTER","w") - lcfile.write(str(counter)) - lcfile.close() - - # open CONTENTS file (possibly overwriting old one) for recording - outfile=open(self.dbtmpdir+"/CONTENTS","w") - - self.updateprotect() - - #if we have a file containing previously-merged config file md5sums, grab it. - conf_mem_file = os.path.join(destroot, CONFIG_MEMORY_FILE) - cfgfiledict = grabdict(conf_mem_file) - if self.settings.has_key("NOCONFMEM"): - cfgfiledict["IGNORE"]=1 - else: - cfgfiledict["IGNORE"]=0 - - # set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change) - mymtime = long(time.time()) - prevmask = os.umask(0) - secondhand = [] - - # we do a first merge; this will recurse through all files in our srcroot but also build up a - # "second hand" of symlinks to merge later - if self.mergeme(srcroot,destroot,outfile,secondhand,EPREFIX.lstrip(os.path.sep),cfgfiledict,mymtime): - return 1 - - # now, it's time for dealing our second hand; we'll loop until we can't merge anymore. The rest are - # broken symlinks. We'll merge them too. - lastlen=0 - while len(secondhand) and len(secondhand)!=lastlen: - # clear the thirdhand. Anything from our second hand that - # couldn't get merged will be added to thirdhand. - - thirdhand=[] - self.mergeme(srcroot,destroot,outfile,thirdhand,secondhand,cfgfiledict,mymtime) - - #swap hands - lastlen=len(secondhand) - - # our thirdhand now becomes our secondhand. It's ok to throw - # away secondhand since thirdhand contains all the stuff that - # couldn't be merged. - secondhand = thirdhand - - if len(secondhand): - # force merge of remaining symlinks (broken or circular; oh well) - self.mergeme(srcroot,destroot,outfile,None,secondhand,cfgfiledict,mymtime) - - #restore umask - os.umask(prevmask) - - #if we opened it, close it - outfile.flush() - outfile.close() - - if os.path.exists(self.dbpkgdir): - writemsg_stdout(">>> Safely unmerging already-installed instance...\n") - self.dbdir = self.dbpkgdir - self.unmerge(oldcontents, trimworld=0, ldpath_mtimes=prev_mtimes) - self.dbdir = self.dbtmpdir - writemsg_stdout(">>> Original instance of package unmerged safely.\n") - - # We hold both directory locks. - self.dbdir = self.dbpkgdir - self.delete() - movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) - contents = self.getcontents() - - #write out our collection of md5sums - if cfgfiledict.has_key("IGNORE"): - del cfgfiledict["IGNORE"] - - my_private_path = os.path.join(destroot, PRIVATE_PATH) - if not os.path.exists(my_private_path): - os.makedirs(my_private_path) - os.chown(my_private_path, os.getuid(), portage_gid) - os.chmod(my_private_path, 02770) - - writedict(cfgfiledict, conf_mem_file) - del conf_mem_file - - #do postinst script - a = doebuild(myebuild, "postinst", destroot, self.settings, use_cache=0, - tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) - - # XXX: Decide how to handle failures here. - if a != os.EX_OK: - writemsg("!!! FAILED postinst: "+str(a)+"\n", noiselevel=-1) - return a - - downgrade = False - for v in otherversions: - if pkgcmp(catpkgsplit(self.pkg)[1:], catpkgsplit(v)[1:]) < 0: - downgrade = True - - #update environment settings, library paths. DO NOT change symlinks. - env_update(makelinks=(not downgrade), - target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes, - contents=contents) - #dircache may break autoclean because it remembers the -MERGING-pkg file - global dircache - if dircache.has_key(self.dbcatdir): - del dircache[self.dbcatdir] - writemsg_stdout(">>> %s %s\n" % (self.mycpv,"merged.")) - - # Process ebuild logfiles - elog_process(self.mycpv, self.settings) - if "noclean" not in self.settings.features: - doebuild(myebuild, "clean", destroot, self.settings, - tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) - return os.EX_OK - - def mergeme(self,srcroot,destroot,outfile,secondhand,stufftomerge,cfgfiledict,thismtime): - """ - - This function handles actual merging of the package contents to the livefs. - It also handles config protection. - - @param srcroot: Where are we copying files from (usually ${D}) - @type srcroot: String (Path) - @param destroot: Typically ${ROOT} - @type destroot: String (Path) - @param outfile: File to log operations to - @type outfile: File Object - @param secondhand: A set of items to merge in pass two (usually - or symlinks that point to non-existing files that may get merged later) - @type secondhand: List - @param stufftomerge: Either a diretory to merge, or a list of items. - @type stufftomerge: String or List - @param cfgfiledict: { File:mtime } mapping for config_protected files - @type cfgfiledict: Dictionary - @param thismtime: The current time (typically long(time.time()) - @type thismtime: Long - @rtype: None or Boolean - @returns: - 1. True on failure - 2. None otherwise - - """ - from os.path import sep, join - srcroot = normalize_path(srcroot).rstrip(sep) + sep - destroot = normalize_path(destroot).rstrip(sep) + sep - # this is supposed to merge a list of files. There will be 2 forms of argument passing. - if type(stufftomerge)==types.StringType: - #A directory is specified. Figure out protection paths, listdir() it and process it. - mergelist = listdir(join(srcroot, stufftomerge)) - offset=stufftomerge - else: - mergelist=stufftomerge - offset="" - for x in mergelist: - mysrc = join(srcroot, offset, x) - mydest = join(destroot, offset, x) - # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") - myrealdest = join(sep, offset, x) - # stat file once, test using S_* macros many times (faster that way) - try: - mystat=os.lstat(mysrc) - except OSError, e: - writemsg("\n") - writemsg(red("!!! ERROR: There appears to be ")+bold("FILE SYSTEM CORRUPTION.")+red(" A file that is listed\n")) - writemsg(red("!!! as existing is not capable of being stat'd. If you are using an\n")) - writemsg(red("!!! experimental kernel, please boot into a stable one, force an fsck,\n")) - writemsg(red("!!! and ensure your filesystem is in a sane state. ")+bold("'shutdown -Fr now'\n")) - writemsg(red("!!! File: ")+str(mysrc)+"\n", noiselevel=-1) - writemsg(red("!!! Error: ")+str(e)+"\n", noiselevel=-1) - sys.exit(1) - except Exception, e: - writemsg("\n") - writemsg(red("!!! ERROR: An unknown error has occurred during the merge process.\n")) - writemsg(red("!!! A stat call returned the following error for the following file:")) - writemsg( "!!! Please ensure that your filesystem is intact, otherwise report\n") - writemsg( "!!! this as a portage bug at bugs.gentoo.org. Append 'emerge info'.\n") - writemsg( "!!! File: "+str(mysrc)+"\n", noiselevel=-1) - writemsg( "!!! Error: "+str(e)+"\n", noiselevel=-1) - sys.exit(1) - - - mymode=mystat[stat.ST_MODE] - # handy variables; mydest is the target object on the live filesystems; - # mysrc is the source object in the temporary install dir - try: - mydmode = os.lstat(mydest).st_mode - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e - #dest file doesn't exist - mydmode=None - - if stat.S_ISLNK(mymode): - # we are merging a symbolic link - myabsto=abssymlink(mysrc) - if myabsto.startswith(srcroot): - myabsto=myabsto[len(srcroot):] - myabsto = myabsto.lstrip(sep) - myto=os.readlink(mysrc) - if self.settings and self.settings["D"]: - if myto.startswith(self.settings["D"]): - myto=myto[len(self.settings["D"]):] - # myrealto contains the path of the real file to which this symlink points. - # we can simply test for existence of this file to see if the target has been merged yet - myrealto = normalize_path(os.path.join(destroot, myabsto)) - if mydmode!=None: - #destination exists - if not stat.S_ISLNK(mydmode): - if stat.S_ISDIR(mydmode): - # directory in the way: we can't merge a symlink over a directory - # we won't merge this, continue with next file... - continue - - if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]): - # Kill file blocking installation of symlink to dir #71787 - pass - elif self.isprotected(mydest): - # Use md5 of the target in ${D} if it exists... - try: - newmd5 = portage.checksum.perform_md5( - join(srcroot, myabsto)) - except portage.exception.FileNotFound: - # Maybe the target is merged already. - try: - newmd5 = portage.checksum.perform_md5( - myrealto) - except portage.exception.FileNotFound: - newmd5 = None - mydest = new_protect_filename(mydest,newmd5=newmd5) - - # if secondhand is None it means we're operating in "force" mode and should not create a second hand. - if (secondhand!=None) and (not os.path.exists(myrealto)): - # either the target directory doesn't exist yet or the target file doesn't exist -- or - # the target is a broken symlink. We will add this file to our "second hand" and merge - # it later. - secondhand.append(mysrc[len(srcroot):]) - continue - # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly - mymtime=movefile(mysrc,mydest,newmtime=thismtime,sstat=mystat, mysettings=self.settings) - if mymtime!=None: - writemsg_stdout(">>> %s -> %s\n" % (mydest, myto)) - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") - else: - print "!!! Failed to move file." - print "!!!",mydest,"->",myto - sys.exit(1) - elif stat.S_ISDIR(mymode): - # we are merging a directory - if mydmode!=None: - # destination exists - - if bsd_chflags: - # Save then clear flags on dest. - dflags = os.lstat(mydest).st_flags - if dflags != 0: - bsd_chflags.lchflags(mydest, 0) - - if not os.access(mydest, os.W_OK): - pkgstuff = pkgsplit(self.pkg) - writemsg("\n!!! Cannot write to '"+mydest+"'.\n", noiselevel=-1) - writemsg("!!! Please check permissions and directories for broken symlinks.\n") - writemsg("!!! You may start the merge process again by using ebuild:\n") - writemsg("!!! ebuild "+self.settings["PORTDIR"]+"/"+self.cat+"/"+pkgstuff[0]+"/"+self.pkg+".ebuild merge\n") - writemsg("!!! And finish by running this: env-update\n\n") - return 1 - - if stat.S_ISLNK(mydmode) or stat.S_ISDIR(mydmode): - # a symlink to an existing directory will work for us; keep it: - writemsg_stdout("--- %s/\n" % mydest) - if bsd_chflags: - bsd_chflags.lchflags(mydest, dflags) - else: - # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. - if movefile(mydest,mydest+".backup", mysettings=self.settings) is None: - sys.exit(1) - print "bak",mydest,mydest+".backup" - #now create our directory - if self.settings.selinux_enabled(): - sid = selinux.get_sid(mysrc) - selinux.secure_mkdir(mydest,sid) - else: - os.mkdir(mydest) - if bsd_chflags: - bsd_chflags.lchflags(mydest, dflags) - os.chmod(mydest,mystat[0]) - os.chown(mydest,mystat[4],mystat[5]) - writemsg_stdout(">>> %s/\n" % mydest) - else: - #destination doesn't exist - if self.settings.selinux_enabled(): - sid = selinux.get_sid(mysrc) - selinux.secure_mkdir(mydest,sid) - else: - os.mkdir(mydest) - os.chmod(mydest,mystat[0]) - os.chown(mydest,mystat[4],mystat[5]) - writemsg_stdout(">>> %s/\n" % mydest) - outfile.write("dir "+myrealdest+"\n") - # recurse and merge this directory - if self.mergeme(srcroot, destroot, outfile, secondhand, - join(offset, x), cfgfiledict, thismtime): - return 1 - elif stat.S_ISREG(mymode): - # we are merging a regular file - mymd5=portage.checksum.perform_md5(mysrc,calc_prelink=1) - # calculate config file protection stuff - mydestdir=os.path.dirname(mydest) - moveme=1 - zing="!!!" - if mydmode!=None: - # destination file exists - if stat.S_ISDIR(mydmode): - # install of destination is blocked by an existing directory with the same name - moveme=0 - writemsg_stdout("!!! %s\n" % mydest) - elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): - cfgprot=0 - # install of destination is blocked by an existing regular file, - # or by a symlink to an existing regular file; - # now, config file management may come into play. - # we only need to tweak mydest if cfg file management is in play. - if self.isprotected(mydest): - # we have a protection path; enable config file management. - destmd5=portage.checksum.perform_md5(mydest,calc_prelink=1) - if mymd5==destmd5: - #file already in place; simply update mtimes of destination - os.utime(mydest,(thismtime,thismtime)) - zing="---" - moveme=0 - else: - if mymd5 == cfgfiledict.get(myrealdest, [None])[0]: - """ An identical update has previously been - merged. Skip it unless the user has chosen - --noconfmem.""" - zing = "-o-" - moveme = cfgfiledict["IGNORE"] - cfgprot = cfgfiledict["IGNORE"] - else: - moveme = 1 - cfgprot = 1 - if moveme: - # Merging a new file, so update confmem. - cfgfiledict[myrealdest] = [mymd5] - elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]: - """A previously remembered update has been - accepted, so it is removed from confmem.""" - del cfgfiledict[myrealdest] - if cfgprot: - mydest = new_protect_filename(mydest, newmd5=mymd5) - - # whether config protection or not, we merge the new file the - # same way. Unless moveme=0 (blocking directory) - if moveme: - mymtime=movefile(mysrc,mydest,newmtime=thismtime,sstat=mystat, mysettings=self.settings) - if mymtime is None: - sys.exit(1) - zing=">>>" - else: - mymtime=thismtime - # We need to touch the destination so that on --update the - # old package won't yank the file with it. (non-cfgprot related) - os.utime(mydest,(thismtime,thismtime)) - zing="---" - if self.settings["USERLAND"] == "Darwin" and myrealdest[-2:] == ".a": - - # XXX kludge, can be killed when portage stops relying on - # md5+mtime, and uses refcounts - # alright, we've fooled w/ mtime on the file; this pisses off static archives - # basically internal mtime != file's mtime, so the linker (falsely) thinks - # the archive is stale, and needs to have it's toc rebuilt. - - myf = open(mydest, "r+") - - # ar mtime field is digits padded with spaces, 12 bytes. - lms=str(thismtime+5).ljust(12) - myf.seek(0) - magic=myf.read(8) - if magic != "!\n": - # not an archive (dolib.a from portage.py makes it here fex) - myf.close() - else: - st = os.stat(mydest) - while myf.tell() < st.st_size - 12: - # skip object name - myf.seek(16,1) - - # update mtime - myf.write(lms) - - # skip uid/gid/mperm - myf.seek(20,1) - - # read the archive member's size - x=long(myf.read(10)) - - # skip the trailing newlines, and add the potential - # extra padding byte if it's not an even size - myf.seek(x + 2 + (x % 2),1) - - # and now we're at the end. yay. - myf.close() - mymd5 = portage.checksum.perform_md5(mydest, calc_prelink=1) - os.utime(mydest,(thismtime,thismtime)) - - if mymtime!=None: - zing=">>>" - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") - writemsg_stdout("%s %s\n" % (zing,mydest)) - else: - # we are merging a fifo or device node - zing="!!!" - if mydmode is None: - # destination doesn't exist - if movefile(mysrc,mydest,newmtime=thismtime,sstat=mystat, mysettings=self.settings)!=None: - zing=">>>" - else: - sys.exit(1) - if stat.S_ISFIFO(mymode): - outfile.write("fif %s\n" % myrealdest) - else: - outfile.write("dev %s\n" % myrealdest) - writemsg_stdout(zing+" "+mydest+"\n") - - def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, - mydbapi=None, prev_mtimes=None): - try: - self.lockdb() - return self.treewalk(mergeroot, myroot, inforoot, myebuild, - cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) - finally: - self.unlockdb() - - def getstring(self,name): - "returns contents of a file with whitespace converted to spaces" - if not os.path.exists(self.dbdir+"/"+name): - return "" - myfile=open(self.dbdir+"/"+name,"r") - mydata=myfile.read().split() - myfile.close() - return " ".join(mydata) - - def copyfile(self,fname): - shutil.copyfile(fname,self.dbdir+"/"+os.path.basename(fname)) - - def getfile(self,fname): - if not os.path.exists(self.dbdir+"/"+fname): - return "" - myfile=open(self.dbdir+"/"+fname,"r") - mydata=myfile.read() - myfile.close() - return mydata - - def setfile(self,fname,data): - write_atomic(os.path.join(self.dbdir, fname), data) - - def getelements(self,ename): - if not os.path.exists(self.dbdir+"/"+ename): - return [] - myelement=open(self.dbdir+"/"+ename,"r") - mylines=myelement.readlines() - myreturn=[] - for x in mylines: - for y in x[:-1].split(): - myreturn.append(y) - myelement.close() - return myreturn - - def setelements(self,mylist,ename): - myelement=open(self.dbdir+"/"+ename,"w") - for x in mylist: - myelement.write(x+"\n") - myelement.close() - - def isregular(self): - "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" - return os.path.exists(self.dbdir+"/CATEGORY") +from portage.dbapi import dbapi +from portage.dbapi.virtual import fakedbapi +from portage.dbapi.bintree import bindbapi, binarytree +from portage.dbapi.vartree import vardbapi, vartree, dblink +from portage.dbapi.porttree import close_portdbapi_caches, portdbapi, portagetree class FetchlistDict(UserDict.DictMixin): """This provide a mapping interface to retrieve fetch lists. It's used diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py new file mode 100644 index 000000000..b4ec89e53 --- /dev/null +++ b/pym/portage/dbapi/__init__.py @@ -0,0 +1,55 @@ +from portage.dep import dep_getslot, dep_getkey, match_from_list +from portage.locks import unlockfile +from portage.output import red +from portage.util import writemsg + +from portage import dep_expand + +import os, re + +class dbapi(object): + def __init__(self): + pass + + def close_caches(self): + pass + + def cp_list(self, cp, use_cache=1): + return + + def cpv_all(self): + cpv_list = [] + for cp in self.cp_all(): + cpv_list.extend(self.cp_list(cp)) + return cpv_list + + def aux_get(self, mycpv, mylist): + "stub code for returning auxiliary db information, such as SLOT, DEPEND, etc." + 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' + 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or [] if mycpv not found' + raise NotImplementedError + + def match(self, origdep, use_cache=1): + mydep = dep_expand(origdep, mydb=self, settings=self.settings) + mykey = dep_getkey(mydep) + mylist = match_from_list(mydep, self.cp_list(mykey, use_cache=use_cache)) + myslot = dep_getslot(mydep) + if myslot is not None: + mylist = [cpv for cpv in mylist \ + if self.aux_get(cpv, ["SLOT"])[0] == myslot] + return mylist + + def invalidentry(self, mypath): + if re.search("portage_lockfile$", mypath): + if not os.environ.has_key("PORTAGE_MASTER_PID"): + writemsg("Lockfile removed: %s\n" % mypath, 1) + unlockfile((mypath, None, None)) + else: + # Nothing we can do about it. We're probably sandboxed. + pass + elif re.search(".*/-MERGING-(.*)", mypath): + if os.path.exists(mypath): + writemsg(red("INCOMPLETE MERGE:")+" "+mypath+"\n", noiselevel=-1) + else: + writemsg("!!! Invalid db entry: %s\n" % mypath, noiselevel=-1) + diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py new file mode 100644 index 000000000..a03fb124c --- /dev/null +++ b/pym/portage/dbapi/bintree.py @@ -0,0 +1,522 @@ +from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list +from portage.dbapi.virtual import fakedbapi +from portage.exception import InvalidPackageName, InvalidAtom +from portage.output import green +from portage.util import normalize_path, writemsg, writemsg_stdout +from portage.versions import best, catpkgsplit, catsplit +from portage.update import update_dbentries + +from portage import listdir, dep_expand + +import portage.xpak, portage.getbinpkg + +import os, errno, stat + +class bindbapi(fakedbapi): + def __init__(self, mybintree=None, settings=None): + self.bintree = mybintree + self.cpvdict={} + self.cpdict={} + if settings is None: + from portage import settings + self.settings = settings + self._match_cache = {} + # Selectively cache metadata in order to optimize dep matching. + self._aux_cache_keys = set(["SLOT"]) + self._aux_cache = {} + + def match(self, *pargs, **kwargs): + if self.bintree and not self.bintree.populated: + self.bintree.populate() + return fakedbapi.match(self, *pargs, **kwargs) + + def aux_get(self, mycpv, wants): + if self.bintree and not self.bintree.populated: + self.bintree.populate() + cache_me = False + if not set(wants).difference(self._aux_cache_keys): + aux_cache = self._aux_cache.get(mycpv) + if aux_cache is not None: + return [aux_cache[x] for x in wants] + cache_me = True + mysplit = mycpv.split("/") + mylist = [] + tbz2name = mysplit[1]+".tbz2" + if self.bintree and not self.bintree.isremote(mycpv): + tbz2 = portage.xpak.tbz2(self.bintree.getname(mycpv)) + getitem = tbz2.getfile + else: + getitem = self.bintree.remotepkgs[tbz2name].get + mydata = {} + mykeys = wants + if cache_me: + mykeys = self._aux_cache_keys.union(wants) + for x in mykeys: + myval = getitem(x) + # myval is None if the key doesn't exist + # or the tbz2 is corrupt. + if myval: + mydata[x] = " ".join(myval.split()) + if "EAPI" in mykeys: + if not mydata.setdefault("EAPI", "0"): + mydata["EAPI"] = "0" + if cache_me: + aux_cache = {} + for x in self._aux_cache_keys: + aux_cache[x] = mydata.get(x, "") + self._aux_cache[mycpv] = aux_cache + return [mydata.get(x, "") for x in wants] + + def aux_update(self, cpv, values): + if not self.bintree.populated: + self.bintree.populate() + tbz2path = self.bintree.getname(cpv) + if not os.path.exists(tbz2path): + raise KeyError(cpv) + mytbz2 = portage.xpak.tbz2(tbz2path) + mydata = mytbz2.get_data() + mydata.update(values) + mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) + + def cp_list(self, *pargs, **kwargs): + if not self.bintree.populated: + self.bintree.populate() + return fakedbapi.cp_list(self, *pargs, **kwargs) + + def cpv_all(self): + if not self.bintree.populated: + self.bintree.populate() + return fakedbapi.cpv_all(self) + + +class binarytree(object): + "this tree scans for a list of all packages available in PKGDIR" + def __init__(self, root, pkgdir, virtual=None, settings=None, clone=None): + if clone: + # XXX This isn't cloning. It's an instance of the same thing. + self.root = clone.root + self.pkgdir = clone.pkgdir + self.dbapi = clone.dbapi + self.populated = clone.populated + self.tree = clone.tree + self.remotepkgs = clone.remotepkgs + self.invalids = clone.invalids + self.settings = clone.settings + else: + self.root = root + #self.pkgdir=settings["PKGDIR"] + self.pkgdir = normalize_path(pkgdir) + self.dbapi = bindbapi(self, settings=settings) + self.populated = 0 + self.tree = {} + self.remotepkgs = {} + self.invalids = [] + self.settings = settings + self._pkg_paths = {} + + def move_ent(self, mylist): + if not self.populated: + self.populate() + origcp = mylist[1] + newcp = mylist[2] + # sanity check + for cp in [origcp, newcp]: + if not (isvalidatom(cp) and isjustname(cp)): + raise InvalidPackageName(cp) + origcat = origcp.split("/")[0] + mynewcat = newcp.split("/")[0] + origmatches=self.dbapi.cp_list(origcp) + if not origmatches: + return + for mycpv in origmatches: + + mycpsplit = catpkgsplit(mycpv) + mynewcpv = newcp + "-" + mycpsplit[2] + if mycpsplit[3] != "r0": + mynewcpv += "-" + mycpsplit[3] + myoldpkg = mycpv.split("/")[1] + mynewpkg = mynewcpv.split("/")[1] + + if (mynewpkg != myoldpkg) and os.path.exists(self.getname(mynewcpv)): + writemsg("!!! Cannot update binary: Destination exists.\n", + noiselevel=-1) + writemsg("!!! "+mycpv+" -> "+mynewcpv+"\n", noiselevel=-1) + continue + + tbz2path = self.getname(mycpv) + if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK): + writemsg("!!! Cannot update readonly binary: "+mycpv+"\n", + noiselevel=-1) + continue + + #print ">>> Updating data in:",mycpv + writemsg_stdout("%") + mytbz2 = portage.xpak.tbz2(tbz2path) + mydata = mytbz2.get_data() + updated_items = update_dbentries([mylist], mydata) + mydata.update(updated_items) + mydata["CATEGORY"] = mynewcat+"\n" + if mynewpkg != myoldpkg: + mydata[mynewpkg+".ebuild"] = mydata[myoldpkg+".ebuild"] + del mydata[myoldpkg+".ebuild"] + mydata["PF"] = mynewpkg + "\n" + mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) + + self.dbapi.cpv_remove(mycpv) + del self._pkg_paths[mycpv] + new_path = self.getname(mynewcpv) + self._pkg_paths[mynewcpv] = os.path.join( + *new_path.split(os.path.sep)[-2:]) + if new_path != mytbz2: + try: + os.makedirs(os.path.dirname(new_path)) + except OSError, e: + if e.errno != errno.EEXIST: + raise + del e + os.rename(tbz2path, new_path) + self._remove_symlink(mycpv) + if new_path.split(os.path.sep)[-2] == "All": + self._create_symlink(mynewcpv) + self.dbapi.cpv_inject(mynewcpv) + + return 1 + + def _remove_symlink(self, cpv): + """Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also remove + the ${PKGDIR}/${CATEGORY} directory if empty. The file will not be + removed if os.path.islink() returns False.""" + mycat, mypkg = catsplit(cpv) + mylink = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2") + if os.path.islink(mylink): + """Only remove it if it's really a link so that this method never + removes a real package that was placed here to avoid a collision.""" + os.unlink(mylink) + try: + os.rmdir(os.path.join(self.pkgdir, mycat)) + except OSError, e: + if e.errno not in (errno.ENOENT, + errno.ENOTEMPTY, errno.EEXIST): + raise + del e + + def _create_symlink(self, cpv): + """Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and + ${PKGDIR}/${CATEGORY} directory, if necessary). Any file that may + exist in the location of the symlink will first be removed.""" + mycat, mypkg = catsplit(cpv) + full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2") + try: + os.makedirs(os.path.dirname(full_path)) + except OSError, e: + if e.errno != errno.EEXIST: + raise + del e + try: + os.unlink(full_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + os.symlink(os.path.join("..", "All", mypkg + ".tbz2"), full_path) + + def move_slot_ent(self, mylist): + if not self.populated: + self.populate() + pkg = mylist[1] + origslot = mylist[2] + newslot = mylist[3] + + if not isvalidatom(pkg): + raise InvalidAtom(pkg) + + origmatches = self.dbapi.match(pkg) + if not origmatches: + return + for mycpv in origmatches: + mycpsplit = catpkgsplit(mycpv) + myoldpkg = mycpv.split("/")[1] + tbz2path = self.getname(mycpv) + if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK): + writemsg("!!! Cannot update readonly binary: "+mycpv+"\n", + noiselevel=-1) + continue + + #print ">>> Updating data in:",mycpv + mytbz2 = portage.xpak.tbz2(tbz2path) + mydata = mytbz2.get_data() + + slot = mydata["SLOT"] + if (not slot): + continue + + if (slot[0] != origslot): + continue + + writemsg_stdout("S") + mydata["SLOT"] = newslot+"\n" + mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) + return 1 + + def update_ents(self, update_iter): + if len(update_iter) == 0: + return + if not self.populated: + self.populate() + + for mycpv in self.dbapi.cp_all(): + tbz2path = self.getname(mycpv) + if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK): + writemsg("!!! Cannot update readonly binary: "+mycpv+"\n", + noiselevel=-1) + continue + #print ">>> Updating binary data:",mycpv + writemsg_stdout("*") + mytbz2 = portage.xpak.tbz2(tbz2path) + mydata = mytbz2.get_data() + updated_items = update_dbentries(update_iter, mydata) + if len(updated_items) > 0: + mydata.update(updated_items) + mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) + return 1 + + def prevent_collision(self, cpv): + """Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is safe to + use for a given cpv. If a collision will occur with an existing + package from another category, the existing package will be bumped to + ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist.""" + full_path = self.getname(cpv) + if "All" == full_path.split(os.path.sep)[-2]: + return + """Move a colliding package if it exists. Code below this point only + executes in rare cases.""" + mycat, mypkg = catsplit(cpv) + myfile = mypkg + ".tbz2" + mypath = os.path.join("All", myfile) + dest_path = os.path.join(self.pkgdir, mypath) + if os.path.exists(dest_path): + # For invalid packages, other_cat could be None. + other_cat = portage.xpak.tbz2(dest_path).getfile("CATEGORY") + if other_cat: + other_cat = other_cat.strip() + self._move_from_all(other_cat + "/" + mypkg) + """The file may or may not exist. Move it if necessary and update + internal state for future calls to getname().""" + self._move_to_all(cpv) + + def _move_to_all(self, cpv): + """If the file exists, move it. Whether or not it exists, update state + for future getname() calls.""" + mycat, mypkg = catsplit(cpv) + myfile = mypkg + ".tbz2" + src_path = os.path.join(self.pkgdir, mycat, myfile) + try: + mystat = os.lstat(src_path) + except OSError, e: + mystat = None + if mystat and stat.S_ISREG(mystat.st_mode): + try: + os.makedirs(os.path.join(self.pkgdir, "All")) + except OSError, e: + if e.errno != errno.EEXIST: + raise + del e + os.rename(src_path, os.path.join(self.pkgdir, "All", myfile)) + self._create_symlink(cpv) + self._pkg_paths[cpv] = os.path.join("All", myfile) + + def _move_from_all(self, cpv): + """Move a package from ${PKGDIR}/All/${PF}.tbz2 to + ${PKGDIR}/${CATEGORY}/${PF}.tbz2 and update state from getname calls.""" + self._remove_symlink(cpv) + mycat, mypkg = catsplit(cpv) + myfile = mypkg + ".tbz2" + mypath = os.path.join(mycat, myfile) + dest_path = os.path.join(self.pkgdir, mypath) + try: + os.makedirs(os.path.dirname(dest_path)) + except OSError, e: + if e.errno != errno.EEXIST: + raise + del e + os.rename(os.path.join(self.pkgdir, "All", myfile), dest_path) + self._pkg_paths[cpv] = mypath + + def populate(self, getbinpkgs=0, getbinpkgsonly=0): + "populates the binarytree" + if (not os.path.isdir(self.pkgdir) and not getbinpkgs): + return 0 + if (not os.path.isdir(self.pkgdir+"/All") and not getbinpkgs): + return 0 + + if not getbinpkgsonly: + pkg_paths = {} + dirs = listdir(self.pkgdir, dirsonly=True, EmptyOnError=True) + if "All" in dirs: + dirs.remove("All") + dirs.sort() + dirs.insert(0, "All") + for mydir in dirs: + for myfile in listdir(os.path.join(self.pkgdir, mydir)): + if not myfile.endswith(".tbz2"): + continue + mypath = os.path.join(mydir, myfile) + full_path = os.path.join(self.pkgdir, mypath) + if os.path.islink(full_path): + continue + mytbz2 = portage.xpak.tbz2(full_path) + # For invalid packages, mycat could be None. + mycat = mytbz2.getfile("CATEGORY") + mypf = mytbz2.getfile("PF") + mypkg = myfile[:-5] + if not mycat or not mypf: + #old-style or corrupt package + writemsg("!!! Invalid binary package: '%s'\n" % full_path, + noiselevel=-1) + writemsg("!!! This binary package is not " + \ + "recoverable and should be deleted.\n", + noiselevel=-1) + self.invalids.append(mypkg) + continue + mycat = mycat.strip() + if mycat != mydir and mydir != "All": + continue + if mypkg != mypf.strip(): + continue + mycpv = mycat + "/" + mypkg + if mycpv in pkg_paths: + # All is first, so it's preferred. + continue + pkg_paths[mycpv] = mypath + self.dbapi.cpv_inject(mycpv) + self._pkg_paths = pkg_paths + + if getbinpkgs and not self.settings["PORTAGE_BINHOST"]: + writemsg(red("!!! PORTAGE_BINHOST unset, but use is requested.\n"), + noiselevel=-1) + + if getbinpkgs and \ + self.settings["PORTAGE_BINHOST"] and not self.remotepkgs: + try: + chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"]) + if chunk_size < 8: + chunk_size = 8 + except (ValueError, KeyError): + chunk_size = 3000 + + writemsg(green("Fetching binary packages info...\n")) + self.remotepkgs = portage.getbinpkg.dir_get_metadata( + self.settings["PORTAGE_BINHOST"], chunk_size=chunk_size) + writemsg(green(" -- DONE!\n\n")) + + for mypkg in self.remotepkgs.keys(): + if not self.remotepkgs[mypkg].has_key("CATEGORY"): + #old-style or corrupt package + writemsg("!!! Invalid remote binary package: "+mypkg+"\n", + noiselevel=-1) + del self.remotepkgs[mypkg] + continue + mycat = self.remotepkgs[mypkg]["CATEGORY"].strip() + fullpkg = mycat+"/"+mypkg[:-5] + mykey = dep_getkey(fullpkg) + try: + # invalid tbz2's can hurt things. + #print "cpv_inject("+str(fullpkg)+")" + self.dbapi.cpv_inject(fullpkg) + #print " -- Injected" + except SystemExit, e: + raise + except: + writemsg("!!! Failed to inject remote binary package:"+str(fullpkg)+"\n", + noiselevel=-1) + del self.remotepkgs[mypkg] + continue + self.populated=1 + + def inject(self, cpv): + return self.dbapi.cpv_inject(cpv) + + def exists_specific(self, cpv): + if not self.populated: + self.populate() + return self.dbapi.match( + dep_expand("="+cpv, mydb=self.dbapi, settings=self.settings)) + + def dep_bestmatch(self, mydep): + "compatibility method -- all matches, not just visible ones" + if not self.populated: + self.populate() + writemsg("\n\n", 1) + writemsg("mydep: %s\n" % mydep, 1) + mydep = dep_expand(mydep, mydb=self.dbapi, settings=self.settings) + writemsg("mydep: %s\n" % mydep, 1) + mykey = dep_getkey(mydep) + writemsg("mykey: %s\n" % mykey, 1) + mymatch = best(match_from_list(mydep,self.dbapi.cp_list(mykey))) + writemsg("mymatch: %s\n" % mymatch, 1) + if mymatch is None: + return "" + return mymatch + + def getname(self, pkgname): + """Returns a file location for this package. The default location is + ${PKGDIR}/All/${PF}.tbz2, but will be ${PKGDIR}/${CATEGORY}/${PF}.tbz2 + in the rare event of a collision. The prevent_collision() method can + be called to ensure that ${PKGDIR}/All/${PF}.tbz2 is available for a + specific cpv.""" + if not self.populated: + self.populate() + mycpv = pkgname + mypath = self._pkg_paths.get(mycpv, None) + if mypath: + return os.path.join(self.pkgdir, mypath) + mycat, mypkg = catsplit(mycpv) + mypath = os.path.join("All", mypkg + ".tbz2") + if mypath in self._pkg_paths.values(): + mypath = os.path.join(mycat, mypkg + ".tbz2") + self._pkg_paths[mycpv] = mypath # cache for future lookups + return os.path.join(self.pkgdir, mypath) + + def isremote(self, pkgname): + "Returns true if the package is kept remotely." + mysplit = pkgname.split("/") + remote = (not os.path.exists(self.getname(pkgname))) and self.remotepkgs.has_key(mysplit[1]+".tbz2") + return remote + + def get_use(self, pkgname): + mysplit=pkgname.split("/") + if self.isremote(pkgname): + return self.remotepkgs[mysplit[1]+".tbz2"]["USE"][:].split() + tbz2=portage.xpak.tbz2(self.getname(pkgname)) + return tbz2.getfile("USE").split() + + def gettbz2(self, pkgname): + "fetches the package from a remote site, if necessary." + print "Fetching '"+str(pkgname)+"'" + mysplit = pkgname.split("/") + tbz2name = mysplit[1]+".tbz2" + if not self.isremote(pkgname): + if (tbz2name not in self.invalids): + return + else: + writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n", + noiselevel=-1) + mydest = self.pkgdir + "/All/" + try: + os.makedirs(mydest, 0775) + except (OSError, IOError): + pass + return portage.getbinpkg.file_get( + self.settings["PORTAGE_BINHOST"] + "/" + tbz2name, + mydest, fcmd=self.settings["RESUMECOMMAND"]) + + def getslot(self, mycatpkg): + "Get a slot for a catpkg; assume it exists." + myslot = "" + try: + myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0] + except SystemExit, e: + raise + except Exception, e: + pass + return myslot diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py new file mode 100644 index 000000000..7293b2e43 --- /dev/null +++ b/pym/portage/dbapi/porttree.py @@ -0,0 +1,783 @@ +from portage.cache.cache_errors import CacheError +from portage.const import REPO_NAME_LOC +from portage.data import portage_gid, secpass +from portage.dbapi import dbapi +from portage.dep import use_reduce, paren_reduce, dep_getslot, dep_getkey, \ + match_from_list, match_to_list +from portage.exception import OperationNotPermitted, PortageException, \ + UntrustedSignature, SecurityViolation, InvalidSignature, MissingSignature, \ + FileNotFound +from portage.manifest import Manifest +from portage.output import red +from portage.util import ensure_dirs, writemsg, apply_recursive_permissions +from portage.versions import pkgsplit, catpkgsplit + +import portage.gpg, portage.checksum + +from portage import eclass_cache, auxdbkeys, auxdbkeylen, doebuild, flatten, \ + listdir, dep_expand, eapi_is_supported, key_expand, dep_check, config + +import os, stat, sys + + +class portdbapi(dbapi): + """this tree will scan a portage directory located at root (passed to init)""" + portdbapi_instances = [] + + def __init__(self, porttree_root, mysettings=None): + portdbapi.portdbapi_instances.append(self) + + if mysettings: + self.mysettings = mysettings + else: + from portage import settings + self.mysettings = config(clone=settings) + + # This is strictly for use in aux_get() doebuild calls when metadata + # is generated by the depend phase. It's safest to use a clone for + # this purpose because doebuild makes many changes to the config + # instance that is passed in. + self.doebuild_settings = config(clone=self.mysettings) + + self.manifestVerifyLevel = None + self.manifestVerifier = None + self.manifestCache = {} # {location: [stat, md5]} + self.manifestMissingCache = [] + + if "gpg" in self.mysettings.features: + self.manifestVerifyLevel = portage.gpg.EXISTS + if "strict" in self.mysettings.features: + self.manifestVerifyLevel = portage.gpg.MARGINAL + self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel) + elif "severe" in self.mysettings.features: + self.manifestVerifyLevel = portage.gpg.TRUSTED + self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", requireSignedRing=True, minimumTrust=self.manifestVerifyLevel) + else: + self.manifestVerifier = portage.gpg.FileChecker(self.mysettings["PORTAGE_GPG_DIR"], "gentoo.gpg", minimumTrust=self.manifestVerifyLevel) + + #self.root=settings["PORTDIR"] + self.porttree_root = os.path.realpath(porttree_root) + + self.depcachedir = self.mysettings.depcachedir[:] + + self.tmpfs = self.mysettings["PORTAGE_TMPFS"] + if self.tmpfs and not os.path.exists(self.tmpfs): + self.tmpfs = None + if self.tmpfs and not os.access(self.tmpfs, os.W_OK): + self.tmpfs = None + if self.tmpfs and not os.access(self.tmpfs, os.R_OK): + self.tmpfs = None + + self.eclassdb = eclass_cache.cache(self.porttree_root, + overlays=self.mysettings["PORTDIR_OVERLAY"].split()) + + self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule") + + #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening) + self.xcache = {} + self.frozen = 0 + + self.porttrees = [self.porttree_root] + \ + [os.path.realpath(t) for t in self.mysettings["PORTDIR_OVERLAY"].split()] + self.treemap = {} + for path in self.porttrees: + repo_name_path = os.path.join(path, REPO_NAME_LOC) + try: + repo_name = open(repo_name_path, 'r').readline().strip() + self.treemap[repo_name] = path + except (OSError,IOError): + pass + + self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule") + self.auxdb = {} + self._init_cache_dirs() + # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys + # ~harring + filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys) + if secpass < 1: + from portage.cache import metadata_overlay, volatile + for x in self.porttrees: + db_ro = self.auxdbmodule(self.depcachedir, x, + filtered_auxdbkeys, gid=portage_gid, readonly=True) + self.auxdb[x] = metadata_overlay.database( + self.depcachedir, x, filtered_auxdbkeys, + gid=portage_gid, db_rw=volatile.database, + db_ro=db_ro) + else: + for x in self.porttrees: + # location, label, auxdbkeys + self.auxdb[x] = self.auxdbmodule( + self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid) + # Selectively cache metadata in order to optimize dep matching. + self._aux_cache_keys = set(["EAPI", "KEYWORDS", "SLOT"]) + self._aux_cache = {} + + def _init_cache_dirs(self): + """Create /var/cache/edb/dep and adjust permissions for the portage + group.""" + + dirmode = 02070 + filemode = 060 + modemask = 02 + + try: + for mydir in (self.depcachedir,): + if ensure_dirs(mydir, gid=portage_gid, mode=dirmode, mask=modemask): + writemsg("Adjusting permissions recursively: '%s'\n" % mydir, + noiselevel=-1) + def onerror(e): + raise # bail out on the first error that occurs during recursion + if not apply_recursive_permissions(mydir, + gid=portage_gid, dirmode=dirmode, dirmask=modemask, + filemode=filemode, filemask=modemask, onerror=onerror): + raise OperationNotPermitted( + "Failed to apply recursive permissions for the portage group.") + except PortageException, e: + pass + + def close_caches(self): + for x in self.auxdb.keys(): + self.auxdb[x].sync() + self.auxdb.clear() + + def flush_cache(self): + for x in self.auxdb.values(): + x.sync() + + def finddigest(self,mycpv): + try: + mydig = self.findname2(mycpv)[0] + if not mydig: + return "" + mydigs = mydig.split("/")[:-1] + mydig = "/".join(mydigs) + mysplit = mycpv.split("/") + except OSError: + return "" + return mydig+"/files/digest-"+mysplit[-1] + + def findname(self,mycpv): + return self.findname2(mycpv)[0] + + def getRepositoryPath(self, repository_id): + """ + This function is required for GLEP 42 compliance; given a valid repository ID + it must return a path to the repository + TreeMap = { id:path } + """ + if repository_id in self.treemap: + return self.treemap[repository_id] + return None + + def getRepositories(self): + """ + This function is required for GLEP 42 compliance; it will return a list of + repository ID's + TreeMap = {id: path} + """ + return [k for k in self.treemap.keys() if k] + + def findname2(self, mycpv, mytree=None): + """ + Returns the location of the CPV, and what overlay it was in. + Searches overlays first, then PORTDIR; this allows us to return the first + matching file. As opposed to starting in portdir and then doing overlays + second, we would have to exhaustively search the overlays until we found + the file we wanted. + """ + if not mycpv: + return "",0 + mysplit = mycpv.split("/") + psplit = pkgsplit(mysplit[1]) + + if mytree: + mytrees = [mytree] + else: + mytrees = self.porttrees[:] + mytrees.reverse() + if psplit: + for x in mytrees: + file=x+"/"+mysplit[0]+"/"+psplit[0]+"/"+mysplit[1]+".ebuild" + if os.access(file, os.R_OK): + return[file, x] + return None, 0 + + def aux_get(self, mycpv, mylist, mytree=None): + "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." + 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' + 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error' + cache_me = False + if not mytree and not set(mylist).difference(self._aux_cache_keys): + aux_cache = self._aux_cache.get(mycpv) + if aux_cache is not None: + return [aux_cache[x] for x in mylist] + cache_me = True + global auxdbkeys, auxdbkeylen + cat,pkg = mycpv.split("/", 1) + + myebuild, mylocation = self.findname2(mycpv, mytree) + + if not myebuild: + writemsg("!!! aux_get(): ebuild path for '%(cpv)s' not specified:\n" % {"cpv":mycpv}, + noiselevel=1) + writemsg("!!! %s\n" % myebuild, noiselevel=1) + raise KeyError(mycpv) + + myManifestPath = "/".join(myebuild.split("/")[:-1])+"/Manifest" + if "gpg" in self.mysettings.features: + try: + mys = portage.gpg.fileStats(myManifestPath) + if (myManifestPath in self.manifestCache) and \ + (self.manifestCache[myManifestPath] == mys): + pass + elif self.manifestVerifier: + if not self.manifestVerifier.verify(myManifestPath): + # Verification failed the desired level. + raise UntrustedSignature, "Untrusted Manifest: %(manifest)s" % {"manifest":myManifestPath} + + if ("severe" in self.mysettings.features) and \ + (mys != portage.gpg.fileStats(myManifestPath)): + raise SecurityViolation, "Manifest changed: %(manifest)s" % {"manifest":myManifestPath} + + except InvalidSignature, e: + if ("strict" in self.mysettings.features) or \ + ("severe" in self.mysettings.features): + raise + writemsg("!!! INVALID MANIFEST SIGNATURE DETECTED: %(manifest)s\n" % {"manifest":myManifestPath}) + except MissingSignature, e: + if ("severe" in self.mysettings.features): + raise + if ("strict" in self.mysettings.features): + if myManifestPath not in self.manifestMissingCache: + writemsg("!!! WARNING: Missing signature in: %(manifest)s\n" % {"manifest":myManifestPath}) + self.manifestMissingCache.insert(0,myManifestPath) + except (OSError, FileNotFound), e: + if ("strict" in self.mysettings.features) or \ + ("severe" in self.mysettings.features): + raise SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)} + writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath}, + noiselevel=-1) + + + if os.access(myebuild, os.R_OK): + emtime = os.stat(myebuild)[stat.ST_MTIME] + else: + writemsg("!!! aux_get(): ebuild for '%(cpv)s' does not exist at:\n" % {"cpv":mycpv}, + noiselevel=-1) + writemsg("!!! %s\n" % myebuild, + noiselevel=-1) + raise KeyError + + try: + mydata = self.auxdb[mylocation][mycpv] + if emtime != long(mydata.get("_mtime_", 0)): + doregen = True + elif len(mydata.get("_eclasses_", [])) > 0: + doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"]) + else: + doregen = False + + except KeyError: + doregen = True + except CacheError: + doregen = True + try: + del self.auxdb[mylocation][mycpv] + except KeyError: + pass + + writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2) + + if doregen: + writemsg("doregen: %s %s\n" % (doregen, mycpv), 2) + writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n", 1) + + self.doebuild_settings.reset() + mydata = {} + myret = doebuild(myebuild, "depend", + self.doebuild_settings["ROOT"], self.doebuild_settings, + dbkey=mydata, tree="porttree", mydbapi=self) + if myret != os.EX_OK: + raise KeyError(mycpv) + + if "EAPI" not in mydata or not mydata["EAPI"].strip(): + mydata["EAPI"] = "0" + + if not eapi_is_supported(mydata["EAPI"]): + # if newer version, wipe everything and negate eapi + eapi = mydata["EAPI"] + mydata = {} + map(lambda x: mydata.setdefault(x, ""), auxdbkeys) + mydata["EAPI"] = "-"+eapi + + if mydata.get("INHERITED", False): + mydata["_eclasses_"] = self.eclassdb.get_eclass_data(mydata["INHERITED"].split()) + else: + mydata["_eclasses_"] = {} + + del mydata["INHERITED"] + + mydata["_mtime_"] = emtime + + self.auxdb[mylocation][mycpv] = mydata + + if not mydata.setdefault("EAPI", "0"): + mydata["EAPI"] = "0" + + #finally, we look at our internal cache entry and return the requested data. + returnme = [] + for x in mylist: + if x == "INHERITED": + returnme.append(' '.join(mydata.get("_eclasses_", {}).keys())) + else: + returnme.append(mydata.get(x,"")) + + if cache_me: + aux_cache = {} + for x in self._aux_cache_keys: + aux_cache[x] = mydata.get(x, "") + self._aux_cache[mycpv] = aux_cache + + return returnme + + def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None): + if mysettings is None: + mysettings = self.mysettings + try: + myuris = self.aux_get(mypkg, ["SRC_URI"], mytree=mytree)[0] + except KeyError: + print red("getfetchlist():")+" aux_get() error reading "+mypkg+"; aborting." + sys.exit(1) + + if useflags is None: + useflags = mysettings["USE"].split() + + myurilist = paren_reduce(myuris) + myurilist = use_reduce(myurilist, uselist=useflags, matchall=all) + newuris = flatten(myurilist) + + myfiles = [] + for x in newuris: + mya = os.path.basename(x) + if not mya in myfiles: + myfiles.append(mya) + return [newuris, myfiles] + + def getfetchsizes(self, mypkg, useflags=None, debug=0): + # returns a filename:size dictionnary of remaining downloads + myebuild = self.findname(mypkg) + pkgdir = os.path.dirname(myebuild) + mf = Manifest(pkgdir, self.mysettings["DISTDIR"]) + checksums = mf.getDigests() + if not checksums: + if debug: + print "[empty/missing/bad digest]: "+mypkg + return None + filesdict={} + if useflags is None: + myuris, myfiles = self.getfetchlist(mypkg,all=1) + else: + myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags) + #XXX: maybe this should be improved: take partial downloads + # into account? check checksums? + for myfile in myfiles: + if myfile not in checksums: + if debug: + writemsg("[bad digest]: missing %s for %s\n" % (myfile, mypkg)) + continue + file_path = os.path.join(self.mysettings["DISTDIR"], myfile) + mystat = None + try: + mystat = os.stat(file_path) + except OSError, e: + pass + if mystat is None: + existing_size = 0 + else: + existing_size = mystat.st_size + remaining_size = int(checksums[myfile]["size"]) - existing_size + if remaining_size > 0: + # Assume the download is resumable. + filesdict[myfile] = remaining_size + elif remaining_size < 0: + # The existing file is too large and therefore corrupt. + filesdict[myfile] = int(checksums[myfile]["size"]) + return filesdict + + def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False): + if not useflags: + if mysettings: + useflags = mysettings["USE"].split() + myuri, myfiles = self.getfetchlist(mypkg, useflags=useflags, mysettings=mysettings, all=all) + myebuild = self.findname(mypkg) + pkgdir = os.path.dirname(myebuild) + mf = Manifest(pkgdir, self.mysettings["DISTDIR"]) + mysums = mf.getDigests() + + failures = {} + for x in myfiles: + if not mysums or x not in mysums: + ok = False + reason = "digest missing" + else: + try: + ok, reason = portage.checksum.verify_all( + os.path.join(self.mysettings["DISTDIR"], x), mysums[x]) + except FileNotFound, e: + ok = False + reason = "File Not Found: '%s'" % str(e) + if not ok: + failures[x] = reason + if failures: + return False + return True + + def getsize(self, mypkg, useflags=None, debug=0): + # returns the total size of remaining downloads + # + # we use getfetchsizes() now, so this function would be obsoleted + # + filesdict = self.getfetchsizes(mypkg, useflags=useflags, debug=debug) + if filesdict is None: + return "[empty/missing/bad digest]" + mysize=0 + for myfile in filesdict.keys(): + mysum += filesdict[myfile] + return mysum + + def cpv_exists(self, mykey): + "Tells us whether an actual ebuild exists on disk (no masking)" + cps2 = mykey.split("/") + cps = catpkgsplit(mykey, silent=0) + if not cps: + #invalid cat/pkg-v + return 0 + if self.findname(cps[0] + "/" + cps2[1]): + return 1 + else: + return 0 + + def cp_all(self): + "returns a list of all keys in our tree" + d = {} + for x in self.mysettings.categories: + for oroot in self.porttrees: + for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1): + d[x+"/"+y] = None + l = d.keys() + l.sort() + return l + + def p_list(self,mycp): + d={} + for oroot in self.porttrees: + for x in listdir(oroot+"/"+mycp,EmptyOnError=1,ignorecvs=1): + if x[-7:]==".ebuild": + d[x[:-7]] = None + return d.keys() + + def cp_list(self, mycp, use_cache=1, mytree=None): + mysplit = mycp.split("/") + d={} + if mytree: + mytrees = [mytree] + else: + mytrees = self.porttrees + for oroot in mytrees: + for x in listdir(oroot+"/"+mycp, EmptyOnError=1, ignorecvs=1): + if x.endswith(".ebuild"): + pf = x[:-7] + ps = pkgsplit(pf) + if not ps: + writemsg("\nInvalid ebuild name: %s\n" % \ + os.path.join(oroot, mycp, x), noiselevel=-1) + continue + d[mysplit[0]+"/"+pf] = None + return d.keys() + + def freeze(self): + for x in ["list-visible", "bestmatch-visible", "match-visible", "match-all"]: + self.xcache[x]={} + self.frozen=1 + + def melt(self): + self.xcache = {} + self.frozen = 0 + + def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None): + "caching match function; very trick stuff" + #if no updates are being made to the tree, we can consult our xcache... + if self.frozen: + try: + return self.xcache[level][origdep][:] + except KeyError: + pass + + if not mydep: + #this stuff only runs on first call of xmatch() + #create mydep, mykey from origdep + mydep = dep_expand(origdep, mydb=self, settings=self.mysettings) + mykey = dep_getkey(mydep) + + if level == "list-visible": + #a list of all visible packages, not called directly (just by xmatch()) + #myval = self.visible(self.cp_list(mykey)) + + myval = self.gvisible(self.visible(self.cp_list(mykey))) + elif level == "bestmatch-visible": + #dep match -- best match of all visible packages + #get all visible matches (from xmatch()), then choose the best one + + myval = best(self.xmatch("match-visible", None, mydep=mydep, mykey=mykey)) + elif level == "bestmatch-list": + #dep match -- find best match but restrict search to sublist + #no point in calling xmatch again since we're not caching list deps + + myval = best(match_from_list(mydep, mylist)) + elif level == "match-list": + #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible()) + + myval = match_from_list(mydep, mylist) + elif level == "match-visible": + #dep match -- find all visible matches + #get all visible packages, then get the matching ones + + myval = match_from_list(mydep, + self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey)) + elif level == "match-all": + #match *all* visible *and* masked packages + + myval = match_from_list(mydep, self.cp_list(mykey)) + else: + print "ERROR: xmatch doesn't handle", level, "query!" + raise KeyError + myslot = dep_getslot(mydep) + if myslot is not None: + slotmatches = [] + for cpv in myval: + try: + if self.aux_get(cpv, ["SLOT"])[0] == myslot: + slotmatches.append(cpv) + except KeyError: + pass # ebuild masked by corruption + myval = slotmatches + if self.frozen and (level not in ["match-list", "bestmatch-list"]): + self.xcache[level][mydep] = myval + if origdep and origdep != mydep: + self.xcache[level][origdep] = myval + return myval[:] + + def match(self, mydep, use_cache=1): + return self.xmatch("match-visible", mydep) + + def visible(self, mylist): + """two functions in one. Accepts a list of cpv values and uses the package.mask *and* + packages file to remove invisible entries, returning remaining items. This function assumes + that all entries in mylist have the same category and package name.""" + if (mylist is None) or (len(mylist) == 0): + return [] + newlist = mylist[:] + #first, we mask out packages in the package.mask file + mykey = newlist[0] + cpv = catpkgsplit(mykey) + if not cpv: + #invalid cat/pkg-v + print "visible(): invalid cat/pkg-v:", mykey + return [] + mycp = cpv[0] + "/" + cpv[1] + maskdict = self.mysettings.pmaskdict + unmaskdict = self.mysettings.punmaskdict + if maskdict.has_key(mycp): + for x in maskdict[mycp]: + mymatches = self.xmatch("match-all", x) + if mymatches is None: + #error in package.mask file; print warning and continue: + print "visible(): package.mask entry \"" + x + "\" is invalid, ignoring..." + continue + for y in mymatches: + unmask = 0 + if unmaskdict.has_key(mycp): + for z in unmaskdict[mycp]: + mymatches_unmask = self.xmatch("match-all",z) + if y in mymatches_unmask: + unmask = 1 + break + if unmask == 0: + try: + newlist.remove(y) + except ValueError: + pass + + revmaskdict = self.mysettings.prevmaskdict + if revmaskdict.has_key(mycp): + for x in revmaskdict[mycp]: + #important: only match against the still-unmasked entries... + #notice how we pass "newlist" to the xmatch() call below.... + #Without this, ~ deps in the packages files are broken. + mymatches = self.xmatch("match-list", x, mylist=newlist) + if mymatches is None: + #error in packages file; print warning and continue: + print "emerge: visible(): profile packages entry \""+x+"\" is invalid, ignoring..." + continue + pos = 0 + while pos < len(newlist): + if newlist[pos] not in mymatches: + del newlist[pos] + else: + pos += 1 + return newlist + + def gvisible(self,mylist): + "strip out group-masked (not in current group) entries" + + if mylist is None: + return [] + newlist=[] + + accept_keywords = self.mysettings["ACCEPT_KEYWORDS"].split() + pkgdict = self.mysettings.pkeywordsdict + for mycpv in mylist: + try: + keys, eapi = self.aux_get(mycpv, ["KEYWORDS", "EAPI"]) + except KeyError: + continue + except PortageException, e: + writemsg("!!! Error: aux_get('%s', ['KEYWORDS', 'EAPI'])\n" % \ + mycpv, noiselevel=-1) + writemsg("!!! %s\n" % str(e), noiselevel=-1) + del e + continue + mygroups = keys.split() + # Repoman may modify this attribute as necessary. + pgroups = accept_keywords[:] + match=0 + cp = dep_getkey(mycpv) + if pkgdict.has_key(cp): + matches = match_to_list(mycpv, pkgdict[cp].keys()) + for atom in matches: + pgroups.extend(pkgdict[cp][atom]) + if matches: + inc_pgroups = [] + for x in pgroups: + # The -* special case should be removed once the tree + # is clean of KEYWORDS=-* crap + if x != "-*" and x.startswith("-"): + try: + inc_pgroups.remove(x[1:]) + except ValueError: + pass + if x not in inc_pgroups: + inc_pgroups.append(x) + pgroups = inc_pgroups + del inc_pgroups + hasstable = False + hastesting = False + for gp in mygroups: + if gp=="*": + writemsg("--- WARNING: Package '%s' uses '*' keyword.\n" % mycpv, + noiselevel=-1) + match=1 + break + elif gp in pgroups: + match=1 + break + elif gp[0] == "~": + hastesting = True + elif gp[0] != "-": + hasstable = True + if not match and ((hastesting and "~*" in pgroups) or (hasstable and "*" in pgroups) or "**" in pgroups): + match=1 + if match and eapi_is_supported(eapi): + newlist.append(mycpv) + return newlist + + +def close_portdbapi_caches(): + for i in portdbapi.portdbapi_instances: + i.close_caches() + + +class portagetree(object): + def __init__(self, root="/", virtual=None, clone=None, settings=None): + """ + Constructor for a PortageTree + + @param root: ${ROOT}, defaults to '/', see make.conf(5) + @type root: String/Path + @param virtual: UNUSED + @type virtual: No Idea + @param clone: Set this if you want a copy of Clone + @type clone: Existing portagetree Instance + @param settings: Portage Configuration object (portage.settings) + @type settings: Instance of portage.config + """ + + if clone: + self.root = clone.root + self.portroot = clone.portroot + self.pkglines = clone.pkglines + else: + self.root = root + if settings is None: + from portage import settings + self.settings = settings + self.portroot = settings["PORTDIR"] + self.virtual = virtual + self.dbapi = portdbapi( + settings["PORTDIR"], mysettings=settings) + + def dep_bestmatch(self,mydep): + "compatibility method" + mymatch = self.dbapi.xmatch("bestmatch-visible",mydep) + if mymatch is None: + return "" + return mymatch + + def dep_match(self,mydep): + "compatibility method" + mymatch = self.dbapi.xmatch("match-visible",mydep) + if mymatch is None: + return [] + return mymatch + + def exists_specific(self,cpv): + return self.dbapi.cpv_exists(cpv) + + def getallnodes(self): + """new behavior: these are all *unmasked* nodes. There may or may not be available + masked package for nodes in this nodes list.""" + return self.dbapi.cp_all() + + def getname(self, pkgname): + "returns file location for this particular package (DEPRECATED)" + if not pkgname: + return "" + mysplit = pkgname.split("/") + psplit = pkgsplit(mysplit[1]) + return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild" + + def resolve_specific(self, myspec): + cps = catpkgsplit(myspec) + if not cps: + return None + mykey = key_expand(cps[0]+"/"+cps[1], mydb=self.dbapi, + settings=self.settings) + mykey = mykey + "-" + cps[2] + if cps[3] != "r0": + mykey = mykey + "-" + cps[3] + return mykey + + def depcheck(self, mycheck, use="yes", myusesplit=None): + return dep_check(mycheck, self.dbapi, use=use, myuse=myusesplit) + + def getslot(self,mycatpkg): + "Get a slot for a catpkg; assume it exists." + myslot = "" + try: + myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] + except SystemExit, e: + raise + except Exception, e: + pass + return myslot + diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py new file mode 100644 index 000000000..9f59432b6 --- /dev/null +++ b/pym/portage/dbapi/vartree.py @@ -0,0 +1,1839 @@ +from portage.checksum import perform_md5 +from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, PRIVATE_PATH, VDB_PATH, EPREFIX +from portage.data import portage_gid, portage_uid, secpass +from portage.dbapi import dbapi +from portage.dep import dep_getslot, use_reduce, paren_reduce, isvalidatom, \ + isjustname, dep_getkey, match_from_list +from portage.elog import elog_process +from portage.exception import InvalidPackageName, InvalidAtom, \ + UnsupportedAPIException, FileNotFound +from portage.locks import lockdir, unlockdir +from portage.output import red, green +from portage.update import fixdbentries +from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, \ + writemsg, writemsg_stdout, write_atomic, atomic_ofstream, writedict, \ + grabfile, grabdict, normalize_path, new_protect_filename +from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp + +from portage import listdir, dep_expand, config, flatten, key_expand, \ + doebuild_environment, doebuild, env_update, dircache, \ + abssymlink, movefile, bsd_chflags + +import os, sys, stat, cPickle, errno, commands, copy, time +from itertools import izip + +class vardbapi(dbapi): + def __init__(self, root, categories=None, settings=None, vartree=None): + self.root = root[:] + + #cache for category directory mtimes + self.mtdircache = {} + + #cache for dependency checks + self.matchcache = {} + + #cache for cp_list results + self.cpcache = {} + + self.blockers = None + if settings is None: + from portage import settings + self.settings = settings + if categories is None: + categories = settings.categories + self.categories = categories[:] + if vartree is None: + from portage import db + vartree = db[root]["vartree"] + self.vartree = vartree + self._aux_cache_keys = set(["SLOT", "COUNTER", "PROVIDE", "USE", + "IUSE", "DEPEND", "RDEPEND", "PDEPEND"]) + self._aux_cache = None + self._aux_cache_version = "1" + self._aux_cache_filename = os.path.join(self.root, + CACHE_PATH.lstrip(os.path.sep), "vdb_metadata.pickle") + + def getpath(self, mykey, filename=None): + rValue = os.path.join(self.root, VDB_PATH, mykey) + if filename != None: + rValue = os.path.join(rValue, filename) + return rValue + + def cpv_exists(self, mykey): + "Tells us whether an actual ebuild exists on disk (no masking)" + return os.path.exists(self.getpath(mykey)) + + def cpv_counter(self, mycpv): + "This method will grab the COUNTER. Returns a counter value." + try: + return long(self.aux_get(mycpv, ["COUNTER"])[0]) + except KeyError, ValueError: + pass + cdir = self.getpath(mycpv) + cpath = self.getpath(mycpv, filename="COUNTER") + + # We write our new counter value to a new file that gets moved into + # place to avoid filesystem corruption on XFS (unexpected reboot.) + corrupted = 0 + if os.path.exists(cpath): + cfile = open(cpath, "r") + try: + counter = long(cfile.readline()) + except ValueError: + print "portage: COUNTER for", mycpv, "was corrupted; resetting to value of 0" + counter = long(0) + corrupted = 1 + cfile.close() + elif os.path.exists(cdir): + mys = pkgsplit(mycpv) + myl = self.match(mys[0], use_cache=0) + print mys, myl + if len(myl) == 1: + try: + # Only one package... Counter doesn't matter. + write_atomic(cpath, "1") + counter = 1 + except SystemExit, e: + raise + except Exception, e: + writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n", + noiselevel=-1) + writemsg("!!! Please run /usr/lib/portage/bin/fix-db.py or\n", + noiselevel=-1) + writemsg("!!! unmerge this exact version.\n", noiselevel=-1) + writemsg("!!! %s\n" % e, noiselevel=-1) + sys.exit(1) + else: + writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n", + noiselevel=-1) + writemsg("!!! Please run /usr/lib/portage/bin/fix-db.py or\n", + noiselevel=-1) + writemsg("!!! remerge the package.\n", noiselevel=-1) + sys.exit(1) + else: + counter = long(0) + if corrupted: + # update new global counter file + write_atomic(cpath, str(counter)) + return counter + + def cpv_inject(self, mycpv): + "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" + os.makedirs(self.getpath(mycpv)) + counter = self.counter_tick(self.root, mycpv=mycpv) + # write local package counter so that emerge clean does the right thing + write_atomic(self.getpath(mycpv, filename="COUNTER"), str(counter)) + + def isInjected(self, mycpv): + if self.cpv_exists(mycpv): + if os.path.exists(self.getpath(mycpv, filename="INJECTED")): + return True + if not os.path.exists(self.getpath(mycpv, filename="CONTENTS")): + return True + return False + + def move_ent(self, mylist): + origcp = mylist[1] + newcp = mylist[2] + + # sanity check + for cp in [origcp, newcp]: + if not (isvalidatom(cp) and isjustname(cp)): + raise InvalidPackageName(cp) + origmatches = self.match(origcp, use_cache=0) + if not origmatches: + return + for mycpv in origmatches: + mycpsplit = catpkgsplit(mycpv) + mynewcpv = newcp + "-" + mycpsplit[2] + mynewcat = newcp.split("/")[0] + if mycpsplit[3] != "r0": + mynewcpv += "-" + mycpsplit[3] + mycpsplit_new = catpkgsplit(mynewcpv) + origpath = self.getpath(mycpv) + if not os.path.exists(origpath): + continue + writemsg_stdout("@") + if not os.path.exists(self.getpath(mynewcat)): + #create the directory + os.makedirs(self.getpath(mynewcat)) + newpath = self.getpath(mynewcpv) + if os.path.exists(newpath): + #dest already exists; keep this puppy where it is. + continue + os.rename(origpath, newpath) + + # We need to rename the ebuild now. + old_pf = catsplit(mycpv)[1] + new_pf = catsplit(mynewcpv)[1] + if new_pf != old_pf: + try: + os.rename(os.path.join(newpath, old_pf + ".ebuild"), + os.path.join(newpath, new_pf + ".ebuild")) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") + + write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") + fixdbentries([mylist], newpath) + + def update_ents(self, update_iter): + """Run fixdbentries on all installed packages (time consuming). Like + fixpackages, this should be run from a helper script and display + a progress indicator.""" + dbdir = os.path.join(self.root, VDB_PATH) + for catdir in listdir(dbdir): + catdir = dbdir+"/"+catdir + if os.path.isdir(catdir): + for pkgdir in listdir(catdir): + pkgdir = catdir+"/"+pkgdir + if os.path.isdir(pkgdir): + fixdbentries(update_iter, pkgdir) + + def move_slot_ent(self, mylist): + pkg = mylist[1] + origslot = mylist[2] + newslot = mylist[3] + + if not isvalidatom(pkg): + raise InvalidAtom(pkg) + + origmatches = self.match(pkg, use_cache=0) + + if not origmatches: + return + for mycpv in origmatches: + origpath = self.getpath(mycpv) + if not os.path.exists(origpath): + continue + + slot = grabfile(os.path.join(origpath, "SLOT")); + if (not slot): + continue + + if (slot[0] != origslot): + continue + + writemsg_stdout("s") + write_atomic(os.path.join(origpath, "SLOT"), newslot+"\n") + + def cp_list(self, mycp, use_cache=1): + mysplit=catsplit(mycp) + if mysplit[0] == '*': + mysplit[0] = mysplit[0][1:] + try: + mystat = os.stat(self.getpath(mysplit[0]))[stat.ST_MTIME] + except OSError: + mystat = 0 + if use_cache and self.cpcache.has_key(mycp): + cpc = self.cpcache[mycp] + if cpc[0] == mystat: + return cpc[1] + list = listdir(self.getpath(mysplit[0]), EmptyOnError=1) + + if (list is None): + return [] + returnme = [] + for x in list: + if x.startswith("."): + continue + if x[0] == '-': + #writemsg(red("INCOMPLETE MERGE:")+str(x[len("-MERGING-"):])+"\n") + continue + ps = pkgsplit(x) + if not ps: + self.invalidentry(os.path.join(self.getpath(mysplit[0]), x)) + continue + if len(mysplit) > 1: + if ps[0] == mysplit[1]: + returnme.append(mysplit[0]+"/"+x) + if use_cache: + self.cpcache[mycp] = [mystat,returnme] + elif self.cpcache.has_key(mycp): + del self.cpcache[mycp] + return returnme + + def cpv_all(self, use_cache=1): + returnme = [] + basepath = os.path.join(self.root, VDB_PATH) + os.path.sep + for x in self.categories: + for y in listdir(basepath + x, EmptyOnError=1): + if y.startswith("."): + continue + subpath = x + "/" + y + # -MERGING- should never be a cpv, nor should files. + if os.path.isdir(basepath + subpath) and (pkgsplit(y) is not None): + returnme += [subpath] + return returnme + + def cp_all(self, use_cache=1): + mylist = self.cpv_all(use_cache=use_cache) + d={} + for y in mylist: + if y[0] == '*': + y = y[1:] + mysplit = catpkgsplit(y) + if not mysplit: + self.invalidentry(self.getpath(y)) + continue + d[mysplit[0]+"/"+mysplit[1]] = None + return d.keys() + + def checkblockers(self, origdep): + pass + + def match(self, origdep, use_cache=1): + "caching match function" + mydep = dep_expand( + origdep, mydb=self, use_cache=use_cache, settings=self.settings) + mykey = dep_getkey(mydep) + mycat = catsplit(mykey)[0] + if not use_cache: + if self.matchcache.has_key(mycat): + del self.mtdircache[mycat] + del self.matchcache[mycat] + mymatch = match_from_list(mydep, + self.cp_list(mykey, use_cache=use_cache)) + myslot = dep_getslot(mydep) + if myslot is not None: + mymatch = [cpv for cpv in mymatch \ + if self.aux_get(cpv, ["SLOT"])[0] == myslot] + return mymatch + try: + curmtime = os.stat(self.root+VDB_PATH+"/"+mycat)[stat.ST_MTIME] + except (IOError, OSError): + curmtime=0 + + if not self.matchcache.has_key(mycat) or not self.mtdircache[mycat]==curmtime: + # clear cache entry + self.mtdircache[mycat] = curmtime + self.matchcache[mycat] = {} + if not self.matchcache[mycat].has_key(mydep): + mymatch = match_from_list(mydep, self.cp_list(mykey, use_cache=use_cache)) + myslot = dep_getslot(mydep) + if myslot is not None: + mymatch = [cpv for cpv in mymatch \ + if self.aux_get(cpv, ["SLOT"])[0] == myslot] + self.matchcache[mycat][mydep] = mymatch + return self.matchcache[mycat][mydep][:] + + def findname(self, mycpv): + return self.getpath(str(mycpv), filename=catsplit(mycpv[1])+".ebuild") + + def flush_cache(self): + """If the current user has permission and the internal aux_get cache has + been updated, save it to disk and mark it unmodified. This is called + by emerge after it has loaded the full vdb for use in dependency + calculations. Currently, the cache is only written if the user has + superuser privileges (since that's required to obtain a lock), but all + users have read access and benefit from faster metadata lookups (as + long as at least part of the cache is still valid).""" + if self._aux_cache is not None and \ + self._aux_cache["modified"] and \ + secpass >= 2: + valid_nodes = set(self.cpv_all()) + for cpv in self._aux_cache["packages"].keys(): + if cpv not in valid_nodes: + del self._aux_cache["packages"][cpv] + del self._aux_cache["modified"] + try: + f = atomic_ofstream(self._aux_cache_filename) + cPickle.dump(self._aux_cache, f, -1) + f.close() + apply_secpass_permissions( + self._aux_cache_filename, gid=portage_gid, mode=0644) + except (IOError, OSError), e: + pass + self._aux_cache["modified"] = False + + def aux_get(self, mycpv, wants): + """This automatically caches selected keys that are frequently needed + by emerge for dependency calculations. The cached metadata is + considered valid if the mtime of the package directory has not changed + since the data was cached. The cache is stored in a pickled dict + object with the following format: + + {version:"1", "packages":{cpv1:(mtime,{k1,v1, k2,v2, ...}), cpv2...}} + + If an error occurs while loading the cache pickle or the version is + unrecognized, the cache will simple be recreated from scratch (it is + completely disposable). + """ + if not self._aux_cache_keys.intersection(wants): + return self._aux_get(mycpv, wants) + if self._aux_cache is None: + try: + f = open(self._aux_cache_filename) + mypickle = cPickle.Unpickler(f) + mypickle.find_global = None + self._aux_cache = mypickle.load() + f.close() + del f + except (IOError, OSError, EOFError, cPickle.UnpicklingError): + pass + if not self._aux_cache or \ + not isinstance(self._aux_cache, dict) or \ + self._aux_cache.get("version") != self._aux_cache_version or \ + not self._aux_cache.get("packages"): + self._aux_cache = {"version": self._aux_cache_version} + self._aux_cache["packages"] = {} + self._aux_cache["modified"] = False + mydir = self.getpath(mycpv) + mydir_stat = None + try: + mydir_stat = os.stat(mydir) + except OSError, e: + if e.errno != errno.ENOENT: + raise + raise KeyError(mycpv) + mydir_mtime = long(mydir_stat.st_mtime) + pkg_data = self._aux_cache["packages"].get(mycpv) + mydata = {} + cache_valid = False + if pkg_data: + cache_mtime, metadata = pkg_data + cache_valid = cache_mtime == mydir_mtime + if cache_valid and set(metadata) != self._aux_cache_keys: + # Allow self._aux_cache_keys to change without a cache version + # bump. + cache_valid = False + if cache_valid: + mydata.update(metadata) + pull_me = set(wants).difference(self._aux_cache_keys) + else: + pull_me = self._aux_cache_keys.union(wants) + if pull_me: + # pull any needed data and cache it + aux_keys = list(pull_me) + for k, v in izip(aux_keys, self._aux_get(mycpv, aux_keys)): + mydata[k] = v + if not cache_valid: + cache_data = {} + for aux_key in self._aux_cache_keys: + cache_data[aux_key] = mydata[aux_key] + self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) + self._aux_cache["modified"] = True + return [mydata[x] for x in wants] + + def _aux_get(self, mycpv, wants): + mydir = self.getpath(mycpv) + try: + if not stat.S_ISDIR(os.stat(mydir).st_mode): + raise KeyError(mycpv) + except OSError, e: + if e.errno == errno.ENOENT: + raise KeyError(mycpv) + del e + raise + results = [] + for x in wants: + try: + myf = open(os.path.join(mydir, x), "r") + try: + myd = myf.read() + finally: + myf.close() + myd = " ".join(myd.split()) + except IOError: + myd = "" + if x == "EAPI" and not myd: + results.append("0") + else: + results.append(myd) + return results + + def aux_update(self, cpv, values): + cat, pkg = catsplit(cpv) + mylink = dblink(cat, pkg, self.root, self.settings, + treetype="vartree", vartree=self.vartree) + if not mylink.exists(): + raise KeyError(cpv) + for k, v in values.iteritems(): + mylink.setfile(k, v) + + def counter_tick(self, myroot, mycpv=None): + return self.counter_tick_core(myroot, incrementing=1, mycpv=mycpv) + + def get_counter_tick_core(self, myroot, mycpv=None): + return self.counter_tick_core(myroot, incrementing=0, mycpv=mycpv) + 1 + + def counter_tick_core(self, myroot, incrementing=1, mycpv=None): + "This method will grab the next COUNTER value and record it back to the global file. Returns new counter value." + cpath = os.path.join(myroot + EPREFIX, CACHE_PATH.lstrip(os.sep), "counter") + changed = 0 + min_counter = 0 + if mycpv: + mysplit = pkgsplit(mycpv) + for x in self.match(mysplit[0], use_cache=0): + if x == mycpv: + continue + try: + old_counter = long(self.aux_get(x, ["COUNTER"])[0]) + writemsg("COUNTER '%d' '%s'\n" % (old_counter, x), 1) + except (ValueError, KeyError): # valueError from long(), KeyError from aux_get + old_counter = 0 + writemsg("!!! BAD COUNTER in '%s'\n" % (x), noiselevel=-1) + if old_counter > min_counter: + min_counter = old_counter + + # We write our new counter value to a new file that gets moved into + # place to avoid filesystem corruption. + find_counter = ("find '%s' -type f -name COUNTER | " + \ + "while read f; do echo $(<\"${f}\"); done | " + \ + "sort -n | tail -n1") % os.path.join(self.root, VDB_PATH) + if os.path.exists(cpath): + cfile = open(cpath, "r") + try: + counter = long(cfile.readline()) + except (ValueError,OverflowError): + try: + counter = long(commands.getoutput(find_counter).strip()) + writemsg("!!! COUNTER was corrupted; resetting to value of %d\n" % counter, + noiselevel=-1) + changed=1 + except (ValueError, OverflowError): + writemsg("!!! COUNTER data is corrupt in pkg db. The values need to be\n", + noiselevel=-1) + writemsg("!!! corrected/normalized so that portage can operate properly.\n", + noiselevel=-1) + writemsg("!!! A simple solution is not yet available so try #gentoo on IRC.\n") + sys.exit(2) + cfile.close() + else: + try: + counter = long(commands.getoutput(find_counter).strip()) + writemsg("!!! Global counter missing. Regenerated from counter files to: %s\n" % counter, + noiselevel=-1) + except ValueError: # Value Error for long(), probably others for commands.getoutput + writemsg("!!! Initializing global counter.\n", noiselevel=-1) + counter = long(0) + changed = 1 + + if counter < min_counter: + counter = min_counter + 1000 + changed = 1 + + if incrementing or changed: + + #increment counter + counter += 1 + # update new global counter file + write_atomic(cpath, str(counter)) + return counter + +class vartree(object): + "this tree will scan a var/db/pkg database located at root (passed to init)" + def __init__(self, root="/", virtual=None, clone=None, categories=None, + settings=None): + if clone: + self.root = clone.root[:] + self.dbapi = copy.deepcopy(clone.dbapi) + self.populated = 1 + self.settings = config(clone=clone.settings) + else: + self.root = root[:] + if settings is None: + from portage import settings + self.settings = settings # for key_expand calls + if categories is None: + categories = settings.categories + self.dbapi = vardbapi(self.root, categories=categories, + settings=settings, vartree=self) + self.populated = 1 + + def getpath(self, mykey, filename=None): + rValue = self.getpath(mykey) + if filename != None: + rValue = os.path.join(rValue, filename) + return rValue + + def zap(self, mycpv): + return + + def inject(self, mycpv): + return + + def get_provide(self, mycpv): + myprovides = [] + mylines = None + try: + mylines, myuse = self.dbapi.aux_get(mycpv, ["PROVIDE", "USE"]) + if mylines: + myuse = myuse.split() + mylines = flatten(use_reduce(paren_reduce(mylines), uselist=myuse)) + for myprovide in mylines: + mys = catpkgsplit(myprovide) + if not mys: + mys = myprovide.split("/") + myprovides += [mys[0] + "/" + mys[1]] + return myprovides + except SystemExit, e: + raise + except Exception, e: + mydir = self.getpath(mycpv) + writemsg("\nParse Error reading PROVIDE and USE in '%s'\n" % mydir, + noiselevel=-1) + if mylines: + writemsg("Possibly Invalid: '%s'\n" % str(mylines), + noiselevel=-1) + writemsg("Exception: %s\n\n" % str(e), noiselevel=-1) + return [] + + def get_all_provides(self): + myprovides = {} + for node in self.getallcpv(): + for mykey in self.get_provide(node): + if myprovides.has_key(mykey): + myprovides[mykey] += [node] + else: + myprovides[mykey] = [node] + return myprovides + + def dep_bestmatch(self, mydep, use_cache=1): + "compatibility method -- all matches, not just visible ones" + #mymatch=best(match(dep_expand(mydep,self.dbapi),self.dbapi)) + mymatch = best(self.dbapi.match( + dep_expand(mydep, mydb=self.dbapi, settings=self.settings), + use_cache=use_cache)) + if mymatch is None: + return "" + else: + return mymatch + + def dep_match(self, mydep, use_cache=1): + "compatibility method -- we want to see all matches, not just visible ones" + #mymatch = match(mydep,self.dbapi) + mymatch = self.dbapi.match(mydep, use_cache=use_cache) + if mymatch is None: + return [] + else: + return mymatch + + def exists_specific(self, cpv): + return self.dbapi.cpv_exists(cpv) + + def getallcpv(self): + """temporary function, probably to be renamed --- Gets a list of all + category/package-versions installed on the system.""" + return self.dbapi.cpv_all() + + def getallnodes(self): + """new behavior: these are all *unmasked* nodes. There may or may not be available + masked package for nodes in this nodes list.""" + return self.dbapi.cp_all() + + def exists_specific_cat(self, cpv, use_cache=1): + cpv = key_expand(cpv, mydb=self.dbapi, use_cache=use_cache, + settings=self.settings) + a = catpkgsplit(cpv) + if not a: + return 0 + mylist = listdir(self.getpath(a[0]), EmptyOnError=1) + for x in mylist: + b = pkgsplit(x) + if not b: + self.dbapi.invalidentry(self.getpath(a[0], filename=x)) + continue + if a[1] == b[0]: + return 1 + return 0 + + def getebuildpath(self, fullpackage): + cat, package = catsplit(fullpackage) + return self.getpath(fullpackage, filename=package+".ebuild") + + def getnode(self, mykey, use_cache=1): + mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, + settings=self.settings) + if not mykey: + return [] + mysplit = catsplit(mykey) + mydirlist = listdir(self.getpath(mysplit[0]),EmptyOnError=1) + returnme = [] + for x in mydirlist: + mypsplit = pkgsplit(x) + if not mypsplit: + self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x)) + continue + if mypsplit[0] == mysplit[1]: + appendme = [mysplit[0]+"/"+x, [mysplit[0], mypsplit[0], mypsplit[1], mypsplit[2]]] + returnme.append(appendme) + return returnme + + + def getslot(self, mycatpkg): + "Get a slot for a catpkg; assume it exists." + try: + return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] + except KeyError: + return "" + + def hasnode(self, mykey, use_cache): + """Does the particular node (cat/pkg key) exist?""" + mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, + settings=self.settings) + mysplit = catsplit(mykey) + mydirlist = listdir(self.getpath(mysplit[0]), EmptyOnError=1) + for x in mydirlist: + mypsplit = pkgsplit(x) + if not mypsplit: + self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x)) + continue + if mypsplit[0] == mysplit[1]: + return 1 + return 0 + + def populate(self): + self.populated=1 + +class dblink(object): + """ + This class provides an interface to the installed package database + At present this is implemented as a text backend in /var/db/pkg. + """ + def __init__(self, cat, pkg, myroot, mysettings, treetype=None, + vartree=None): + """ + Creates a DBlink object for a given CPV. + The given CPV may not be present in the database already. + + @param cat: Category + @type cat: String + @param pkg: Package (PV) + @type pkg: String + @param myroot: Typically ${ROOT} + @type myroot: String (Path) + @param mysettings: Typically portage.config + @type mysettings: An instance of portage.config + @param treetype: one of ['porttree','bintree','vartree'] + @type treetype: String + @param vartree: an instance of vartree corresponding to myroot. + @type vartree: vartree + """ + + self.cat = cat + self.pkg = pkg + self.mycpv = self.cat + "/" + self.pkg + self.mysplit = pkgsplit(self.mycpv) + self.treetype = treetype + if vartree is None: + from portage import db + vartree = db[myroot]["vartree"] + self.vartree = vartree + + self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH)) + self.dbcatdir = self.dbroot+"/"+cat + self.dbpkgdir = self.dbcatdir+"/"+pkg + self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg + self.dbdir = self.dbpkgdir + + self._lock_vdb = None + + self.settings = mysettings + if self.settings == 1: + raise ValueError + + self.myroot=myroot + protect_obj = ConfigProtect(myroot, + mysettings.get("CONFIG_PROTECT","").split(), + mysettings.get("CONFIG_PROTECT_MASK","").split()) + self.updateprotect = protect_obj.updateprotect + self._config_protect = protect_obj + self._installed_instance = None + self.contentscache=[] + self._contents_inodes = None + + def lockdb(self): + if self._lock_vdb: + raise AssertionError("Lock already held.") + # At least the parent needs to exist for the lock file. + ensure_dirs(self.dbroot) + self._lock_vdb = lockdir(self.dbroot) + + def unlockdb(self): + if self._lock_vdb: + unlockdir(self._lock_vdb) + self._lock_vdb = None + + def getpath(self): + "return path to location of db information (for >>> informational display)" + return self.dbdir + + def exists(self): + "does the db entry exist? boolean." + return os.path.exists(self.dbdir) + + def delete(self): + """ + Remove this entry from the database + """ + if not os.path.exists(self.dbdir): + return + try: + for x in listdir(self.dbdir): + os.unlink(self.dbdir+"/"+x) + os.rmdir(self.dbdir) + except OSError, e: + print "!!! Unable to remove db entry for this package." + print "!!! It is possible that a directory is in this one. Portage will still" + print "!!! register this package as installed as long as this directory exists." + print "!!! You may delete this directory with 'rm -Rf "+self.dbdir+"'" + print "!!! "+str(e) + print + sys.exit(1) + + def clearcontents(self): + """ + For a given db entry (self), erase the CONTENTS values. + """ + if os.path.exists(self.dbdir+"/CONTENTS"): + os.unlink(self.dbdir+"/CONTENTS") + + def getcontents(self): + """ + Get the installed files of a given package (aka what that package installed) + """ + contents_file = os.path.join(self.dbdir, "CONTENTS") + if not os.path.exists(contents_file): + return None + if self.contentscache != []: + return self.contentscache + pkgfiles = {} + myc = open(contents_file,"r") + mylines = myc.readlines() + myc.close() + null_byte = "\0" + pos = 0 + for line in mylines: + pos += 1 + if null_byte in line: + # Null bytes are a common indication of corruption. + writemsg("!!! Null byte found in contents " + \ + "file, line %d: '%s'\n" % (pos, contents_file), + noiselevel=-1) + continue + mydat = line.split() + # we do this so we can remove from non-root filesystems + # (use the ROOT var to allow maintenance on other partitions) + try: + mydat[1] = normalize_path(os.path.join( + self.myroot, mydat[1].lstrip(os.path.sep))) + if mydat[0] == "obj": + #format: type, mtime, md5sum + pkgfiles[" ".join(mydat[1:-2])] = [mydat[0], mydat[-1], mydat[-2]] + elif mydat[0] == "dir": + #format: type + pkgfiles[" ".join(mydat[1:])] = [mydat[0] ] + elif mydat[0] == "sym": + #format: type, mtime, dest + x = len(mydat) - 1 + if (x >= 13) and (mydat[-1][-1] == ')'): # Old/Broken symlink entry + mydat = mydat[:-10] + [mydat[-10:][stat.ST_MTIME][:-1]] + writemsg("FIXED SYMLINK LINE: %s\n" % mydat, 1) + x = len(mydat) - 1 + splitter = -1 + while (x >= 0): + if mydat[x] == "->": + splitter = x + break + x = x - 1 + if splitter == -1: + return None + pkgfiles[" ".join(mydat[1:splitter])] = [mydat[0], mydat[-1], " ".join(mydat[(splitter+1):-1])] + elif mydat[0] == "dev": + #format: type + pkgfiles[" ".join(mydat[1:])] = [mydat[0] ] + elif mydat[0]=="fif": + #format: type + pkgfiles[" ".join(mydat[1:])] = [mydat[0]] + else: + return None + except (KeyError, IndexError): + print "portage: CONTENTS line", pos, "corrupt!" + self.contentscache = pkgfiles + return pkgfiles + + def unmerge(self, pkgfiles=None, trimworld=1, cleanup=1, + ldpath_mtimes=None): + """ + Calls prerm + Unmerges a given package (CPV) + calls postrm + calls cleanrm + calls env_update + + @param pkgfiles: files to unmerge (generally self.getcontents() ) + @type pkgfiles: Dictionary + @param trimworld: Remove CPV from world file if True, not if False + @type trimworld: Boolean + @param cleanup: cleanup to pass to doebuild (see doebuild) + @type cleanup: Boolean + @param ldpath_mtimes: mtimes to pass to env_update (see env_update) + @type ldpath_mtimes: Dictionary + @rtype: Integer + @returns: + 1. os.EX_OK if everything went well. + 2. return code of the failed phase (for prerm, postrm, cleanrm) + + Notes: + The caller must ensure that lockdb() and unlockdb() are called + before and after this method. + """ + + contents = self.getcontents() + # Now, don't assume that the name of the ebuild is the same as the + # name of the dir; the package may have been moved. + myebuildpath = None + mystuff = listdir(self.dbdir, EmptyOnError=1) + for x in mystuff: + if x.endswith(".ebuild"): + myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild") + if x[:-7] != self.pkg: + # Clean up after vardbapi.move_ent() breakage in + # portage versions before 2.1.2 + os.rename(os.path.join(self.dbdir, x), myebuildpath) + write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n") + break + + self.settings.load_infodir(self.dbdir) + if myebuildpath: + try: + doebuild_environment(myebuildpath, "prerm", self.myroot, + self.settings, 0, 0, self.vartree.dbapi) + except UnsupportedAPIException, e: + # Sometimes this happens due to corruption of the EAPI file. + writemsg("!!! FAILED prerm: %s\n" % \ + os.path.join(self.dbdir, "EAPI"), noiselevel=-1) + writemsg("%s\n" % str(e), noiselevel=-1) + return 1 + catdir = os.path.dirname(self.settings["PORTAGE_BUILDDIR"]) + ensure_dirs(os.path.dirname(catdir), + uid=portage_uid, gid=portage_gid, mode=070, mask=0) + builddir_lock = None + catdir_lock = None + try: + if myebuildpath: + catdir_lock = lockdir(catdir) + ensure_dirs(catdir, + uid=portage_uid, gid=portage_gid, + mode=070, mask=0) + builddir_lock = lockdir( + self.settings["PORTAGE_BUILDDIR"]) + try: + unlockdir(catdir_lock) + finally: + catdir_lock = None + # Eventually, we'd like to pass in the saved ebuild env here... + retval = doebuild(myebuildpath, "prerm", self.myroot, + self.settings, cleanup=cleanup, use_cache=0, + mydbapi=self.vartree.dbapi, tree="vartree", + vartree=self.vartree) + # XXX: Decide how to handle failures here. + if retval != os.EX_OK: + writemsg("!!! FAILED prerm: %s\n" % retval, noiselevel=-1) + return retval + + self._unmerge_pkgfiles(pkgfiles) + + if myebuildpath: + retval = doebuild(myebuildpath, "postrm", self.myroot, + self.settings, use_cache=0, tree="vartree", + mydbapi=self.vartree.dbapi, vartree=self.vartree) + + # process logs created during pre/postrm + elog_process(self.mycpv, self.settings) + + # XXX: Decide how to handle failures here. + if retval != os.EX_OK: + writemsg("!!! FAILED postrm: %s\n" % retval, noiselevel=-1) + return retval + doebuild(myebuildpath, "cleanrm", self.myroot, self.settings, + tree="vartree", mydbapi=self.vartree.dbapi, + vartree=self.vartree) + + finally: + if builddir_lock: + unlockdir(builddir_lock) + try: + if myebuildpath and not catdir_lock: + # Lock catdir for removal if empty. + catdir_lock = lockdir(catdir) + finally: + if catdir_lock: + try: + os.rmdir(catdir) + except OSError, e: + if e.errno not in (errno.ENOENT, + errno.ENOTEMPTY, errno.EEXIST): + raise + del e + unlockdir(catdir_lock) + env_update(target_root=self.myroot, prev_mtimes=ldpath_mtimes, + contents=contents) + return os.EX_OK + + def _unmerge_pkgfiles(self, pkgfiles): + """ + + Unmerges the contents of a package from the liveFS + Removes the VDB entry for self + + @param pkgfiles: typically self.getcontents() + @type pkgfiles: Dictionary { filename: [ 'type', '?', 'md5sum' ] } + @rtype: None + """ + global dircache + dircache={} + + if not pkgfiles: + writemsg_stdout("No package files given... Grabbing a set.\n") + pkgfiles = self.getcontents() + + if pkgfiles: + mykeys = pkgfiles.keys() + mykeys.sort() + mykeys.reverse() + + #process symlinks second-to-last, directories last. + mydirs = [] + modprotect = "/lib/modules/" + for objkey in mykeys: + obj = normalize_path(objkey) + if obj[:2] == "//": + obj = obj[1:] + statobj = None + try: + statobj = os.stat(obj) + except OSError: + pass + lstatobj = None + try: + lstatobj = os.lstat(obj) + except (OSError, AttributeError): + pass + islink = lstatobj is not None and stat.S_ISLNK(lstatobj.st_mode) + if statobj is None: + if not islink: + #we skip this if we're dealing with a symlink + #because os.stat() will operate on the + #link target rather than the link itself. + writemsg_stdout("--- !found " + str(pkgfiles[objkey][0]) + " %s\n" % obj) + continue + # next line includes a tweak to protect modules from being unmerged, + # but we don't protect modules from being overwritten if they are + # upgraded. We effectively only want one half of the config protection + # functionality for /lib/modules. For portage-ng both capabilities + # should be able to be independently specified. + if obj.startswith(modprotect): + writemsg_stdout("--- cfgpro %s %s\n" % (pkgfiles[objkey][0], obj)) + continue + + lmtime = str(lstatobj[stat.ST_MTIME]) + if (pkgfiles[objkey][0] not in ("dir", "fif", "dev")) and (lmtime != pkgfiles[objkey][1]): + writemsg_stdout("--- !mtime %s %s\n" % (pkgfiles[objkey][0], obj)) + continue + + if pkgfiles[objkey][0] == "dir": + if statobj is None or not stat.S_ISDIR(statobj.st_mode): + writemsg_stdout("--- !dir %s %s\n" % ("dir", obj)) + continue + mydirs.append(obj) + elif pkgfiles[objkey][0] == "sym": + if not islink: + writemsg_stdout("--- !sym %s %s\n" % ("sym", obj)) + continue + try: + os.unlink(obj) + writemsg_stdout("<<< %s %s\n" % ("sym", obj)) + except (OSError, IOError),e: + writemsg_stdout("!!! %s %s\n" % ("sym", obj)) + elif pkgfiles[objkey][0] == "obj": + if statobj is None or not stat.S_ISREG(statobj.st_mode): + writemsg_stdout("--- !obj %s %s\n" % ("obj", obj)) + continue + mymd5 = None + try: + mymd5 = perform_md5(obj, calc_prelink=1) + except FileNotFound, e: + # the file has disappeared between now and our stat call + writemsg_stdout("--- !obj %s %s\n" % ("obj", obj)) + continue + + # string.lower is needed because db entries used to be in upper-case. The + # string.lower allows for backwards compatibility. + if mymd5 != pkgfiles[objkey][2].lower(): + writemsg_stdout("--- !md5 %s %s\n" % ("obj", obj)) + continue + try: + os.unlink(obj) + except (OSError, IOError), e: + pass + writemsg_stdout("<<< %s %s\n" % ("obj", obj)) + elif pkgfiles[objkey][0] == "fif": + if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]): + writemsg_stdout("--- !fif %s %s\n" % ("fif", obj)) + continue + writemsg_stdout("--- %s %s\n" % ("fif", obj)) + elif pkgfiles[objkey][0] == "dev": + writemsg_stdout("--- %s %s\n" % ("dev", obj)) + + mydirs.sort() + mydirs.reverse() + + for obj in mydirs: + try: + os.rmdir(obj) + writemsg_stdout("<<< %s %s\n" % ("dir", obj)) + except (OSError, IOError): + writemsg_stdout("--- !empty dir %s\n" % obj) + + #remove self from vartree database so that our own virtual gets zapped if we're the last node + self.vartree.zap(self.mycpv) + + def isowner(self,filename, destroot): + """ + Check if filename is a new file or belongs to this package + (for this or a previous version) + + @param filename: + @type filename: + @param destroot: + @type destroot: + @rtype: Boolean + @returns: + 1. True if this package owns the file. + 2. False if this package does not own the file. + """ + destfile = normalize_path( + os.path.join(destroot, filename.lstrip(os.path.sep))) + try: + mylstat = os.lstat(destfile) + except (OSError, IOError): + return True + + pkgfiles = self.getcontents() + if pkgfiles and filename in pkgfiles: + return True + if pkgfiles: + if self._contents_inodes is None: + self._contents_inodes = set() + for x in pkgfiles: + try: + lstat = os.lstat(x) + self._contents_inodes.add((lstat.st_dev, lstat.st_ino)) + except OSError: + pass + if (mylstat.st_dev, mylstat.st_ino) in self._contents_inodes: + return True + + return False + + def isprotected(self, filename): + """In cases where an installed package in the same slot owns a + protected file that will be merged, bump the mtime on the installed + file in order to ensure that it isn't unmerged.""" + if not self._config_protect.isprotected(filename): + return False + if self._installed_instance is None: + return True + mydata = self._installed_instance.getcontents().get(filename, None) + if mydata is None: + return True + + # Bump the mtime in order to ensure that the old config file doesn't + # get unmerged. The user will have an opportunity to merge the new + # config with the old one. + try: + os.utime(filename, None) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + # The file has disappeared, so it's not protected. + return False + return True + + + def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0, + mydbapi=None, prev_mtimes=None): + """ + + This function does the following: + + Collision Protection. + calls doebuild(mydo=pkg_preinst) + Merges the package to the livefs + unmerges old version (if required) + calls doebuild(mydo=pkg_postinst) + calls env_update + + @param srcroot: Typically this is ${D} + @type srcroot: String (Path) + @param destroot: Path to merge to (usually ${ROOT}) + @type destroot: String (Path) + @param inforoot: root of the vardb entry ? + @type inforoot: String (Path) + @param myebuild: path to the ebuild that we are processing + @type myebuild: String (Path) + @param mydbapi: dbapi which is handed to doebuild. + @type mydbapi: portdbapi instance + @param prev_mtimes: { Filename:mtime } mapping for env_update + @type prev_mtimes: Dictionary + @rtype: Boolean + @returns: + 1. 0 on success + 2. 1 on failure + + secondhand is a list of symlinks that have been skipped due to their target + not existing; we will merge these symlinks at a later time. + """ + if not os.path.isdir(srcroot): + writemsg("!!! Directory Not Found: D='%s'\n" % srcroot, + noiselevel=-1) + return 1 + + if not os.path.exists(self.dbcatdir): + os.makedirs(self.dbcatdir) + + otherversions = [] + for v in self.vartree.dbapi.cp_list(self.mysplit[0]): + otherversions.append(v.split("/")[1]) + + slot_matches = self.vartree.dbapi.match( + "%s:%s" % (self.mysplit[0], self.settings["SLOT"])) + if slot_matches: + # Used by self.isprotected(). + self._installed_instance = dblink(self.cat, + catsplit(slot_matches[0])[1], destroot, self.settings, + vartree=self.vartree) + + # check for package collisions + if "collision-protect" in self.settings.features: + collision_ignore = set([normalize_path(myignore) for myignore in \ + self.settings.get("COLLISION_IGNORE", "").split()]) + myfilelist = listdir(srcroot, recursive=1, filesonly=1, followSymlinks=False) + + # the linkcheck only works if we are in srcroot + mycwd = os.getcwd() + os.chdir(srcroot) + mysymlinks = filter(os.path.islink, listdir(srcroot, recursive=1, filesonly=0, followSymlinks=False)) + myfilelist.extend(mysymlinks) + mysymlinked_directories = [s + os.path.sep for s in mysymlinks] + del mysymlinks + + stopmerge = False + i=0 + + otherpkg=[] + mypkglist=[] + + if self.pkg in otherversions: + otherversions.remove(self.pkg) # we already checked this package + + myslot = self.settings["SLOT"] + for v in otherversions: + # only allow versions with same slot to overwrite files + if myslot == self.vartree.dbapi.aux_get("/".join((self.cat, v)), ["SLOT"])[0]: + mypkglist.append( + dblink(self.cat, v, destroot, self.settings, + vartree=self.vartree)) + + collisions = [] + + print green("*")+" checking "+str(len(myfilelist))+" files for package collisions" + for f in myfilelist: + nocheck = False + # listdir isn't intelligent enough to exclude symlinked dirs, + # so we have to do it ourself + for s in mysymlinked_directories: + if f.startswith(s): + nocheck = True + break + if nocheck: + continue + i = i + 1 + if i % 1000 == 0: + print str(i)+" files checked ..." + if f[0] != "/": + f="/"+f + isowned = False + for ver in [self] + mypkglist: + if (ver.isowner(f, destroot) or ver.isprotected(f)): + isowned = True + break + if not isowned: + collisions.append(f) + print "existing file "+f+" is not owned by this package" + stopmerge = True + if collision_ignore: + if f in collision_ignore: + stopmerge = False + else: + for myignore in collision_ignore: + if f.startswith(myignore + os.path.sep): + stopmerge = False + break + if stopmerge: + print red("*")+" This package is blocked because it wants to overwrite" + print red("*")+" files belonging to other packages (see messages above)." + print red("*")+" If you have no clue what this is all about report it " + print red("*")+" as a bug for this package on http://bugs.gentoo.org" + print + print red("package "+self.cat+"/"+self.pkg+" NOT merged") + print + print + print "Searching all installed packages for file collisions..." + print "Press Ctrl-C to Stop" + print + """ Note: The isowner calls result in a stat call for *every* + single installed file, since the inode numbers are used to work + around the problem of ambiguous paths caused by symlinked files + and/or directories. Though it is slow, it is as accurate as + possible.""" + found_owner = False + for cpv in self.vartree.dbapi.cpv_all(): + cat, pkg = catsplit(cpv) + mylink = dblink(cat, pkg, destroot, self.settings, + vartree=self.vartree) + mycollisions = [] + for f in collisions: + if mylink.isowner(f, destroot): + mycollisions.append(f) + if mycollisions: + found_owner = True + print " * %s:" % cpv + print + for f in mycollisions: + print " '%s'" % \ + os.path.join(destroot, f.lstrip(os.path.sep)) + print + if not found_owner: + print "None of the installed packages claim the above file(s)." + print + sys.exit(1) + try: + os.chdir(mycwd) + except OSError: + pass + + if os.stat(srcroot).st_dev == os.stat(destroot).st_dev: + """ The merge process may move files out of the image directory, + which causes invalidation of the .installed flag.""" + try: + os.unlink(os.path.join( + os.path.dirname(normalize_path(srcroot)), ".installed")) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + + # get old contents info for later unmerging + oldcontents = self.getcontents() + + self.dbdir = self.dbtmpdir + self.delete() + if not os.path.exists(self.dbtmpdir): + os.makedirs(self.dbtmpdir) + + writemsg_stdout(">>> Merging %s %s %s\n" % (self.mycpv,"to",destroot)) + + # run preinst script + if myebuild is None: + myebuild = os.path.join(inforoot, self.pkg + ".ebuild") + a = doebuild(myebuild, "preinst", destroot, self.settings, cleanup=cleanup, + use_cache=0, tree=self.treetype, mydbapi=mydbapi, + vartree=self.vartree) + + # XXX: Decide how to handle failures here. + if a != os.EX_OK: + writemsg("!!! FAILED preinst: "+str(a)+"\n", noiselevel=-1) + return a + + # copy "info" files (like SLOT, CFLAGS, etc.) into the database + for x in listdir(inforoot): + self.copyfile(inforoot+"/"+x) + + # get current counter value (counter_tick also takes care of incrementing it) + # XXX Need to make this destroot, but it needs to be initialized first. XXX + # XXX bis: leads to some invalidentry() call through cp_all(). + counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) + # write local package counter for recording + lcfile = open(os.path.join(self.dbtmpdir, "COUNTER"),"w") + lcfile.write(str(counter)) + lcfile.close() + + # open CONTENTS file (possibly overwriting old one) for recording + outfile = open(os.path.join(self.dbtmpdir, "CONTENTS"),"w") + + self.updateprotect() + + #if we have a file containing previously-merged config file md5sums, grab it. + conf_mem_file = os.path.join(destroot, CONFIG_MEMORY_FILE) + cfgfiledict = grabdict(conf_mem_file) + if self.settings.has_key("NOCONFMEM"): + cfgfiledict["IGNORE"]=1 + else: + cfgfiledict["IGNORE"]=0 + + # set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change) + mymtime = long(time.time()) + prevmask = os.umask(0) + secondhand = [] + + # we do a first merge; this will recurse through all files in our srcroot but also build up a + # "second hand" of symlinks to merge later + if self.mergeme(srcroot, destroot, outfile, secondhand, EPREFIX.lstrip(os.path.sep), cfgfiledict, mymtime): + return 1 + + # now, it's time for dealing our second hand; we'll loop until we can't merge anymore. The rest are + # broken symlinks. We'll merge them too. + lastlen = 0 + while len(secondhand) and len(secondhand)!=lastlen: + # clear the thirdhand. Anything from our second hand that + # couldn't get merged will be added to thirdhand. + + thirdhand = [] + self.mergeme(srcroot, destroot, outfile, thirdhand, secondhand, cfgfiledict, mymtime) + + #swap hands + lastlen = len(secondhand) + + # our thirdhand now becomes our secondhand. It's ok to throw + # away secondhand since thirdhand contains all the stuff that + # couldn't be merged. + secondhand = thirdhand + + if len(secondhand): + # force merge of remaining symlinks (broken or circular; oh well) + self.mergeme(srcroot, destroot, outfile, None, secondhand, cfgfiledict, mymtime) + + #restore umask + os.umask(prevmask) + + #if we opened it, close it + outfile.flush() + outfile.close() + + if os.path.exists(self.dbpkgdir): + writemsg_stdout(">>> Safely unmerging already-installed instance...\n") + self.dbdir = self.dbpkgdir + self.unmerge(oldcontents, trimworld=0, ldpath_mtimes=prev_mtimes) + self.dbdir = self.dbtmpdir + writemsg_stdout(">>> Original instance of package unmerged safely.\n") + + # We hold both directory locks. + self.dbdir = self.dbpkgdir + self.delete() + movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) + contents = self.getcontents() + + #write out our collection of md5sums + if cfgfiledict.has_key("IGNORE"): + del cfgfiledict["IGNORE"] + + my_private_path = os.path.join(destroot, PRIVATE_PATH) + if not os.path.exists(my_private_path): + os.makedirs(my_private_path) + os.chown(my_private_path, os.getuid(), portage_gid) + os.chmod(my_private_path, 02770) + + writedict(cfgfiledict, conf_mem_file) + del conf_mem_file + + #do postinst script + a = doebuild(myebuild, "postinst", destroot, self.settings, use_cache=0, + tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) + + # XXX: Decide how to handle failures here. + if a != os.EX_OK: + writemsg("!!! FAILED postinst: "+str(a)+"\n", noiselevel=-1) + return a + + downgrade = False + for v in otherversions: + if pkgcmp(catpkgsplit(self.pkg)[1:], catpkgsplit(v)[1:]) < 0: + downgrade = True + + #update environment settings, library paths. DO NOT change symlinks. + env_update(makelinks=(not downgrade), + target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes, + contents=contents) + #dircache may break autoclean because it remembers the -MERGING-pkg file + global dircache + if dircache.has_key(self.dbcatdir): + del dircache[self.dbcatdir] + writemsg_stdout(">>> %s %s\n" % (self.mycpv,"merged.")) + + # Process ebuild logfiles + elog_process(self.mycpv, self.settings) + if "noclean" not in self.settings.features: + doebuild(myebuild, "clean", destroot, self.settings, + tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) + return os.EX_OK + + def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime): + """ + + This function handles actual merging of the package contents to the livefs. + It also handles config protection. + + @param srcroot: Where are we copying files from (usually ${D}) + @type srcroot: String (Path) + @param destroot: Typically ${ROOT} + @type destroot: String (Path) + @param outfile: File to log operations to + @type outfile: File Object + @param secondhand: A set of items to merge in pass two (usually + or symlinks that point to non-existing files that may get merged later) + @type secondhand: List + @param stufftomerge: Either a diretory to merge, or a list of items. + @type stufftomerge: String or List + @param cfgfiledict: { File:mtime } mapping for config_protected files + @type cfgfiledict: Dictionary + @param thismtime: The current time (typically long(time.time()) + @type thismtime: Long + @rtype: None or Boolean + @returns: + 1. True on failure + 2. None otherwise + + """ + from os.path import sep, join + srcroot = normalize_path(srcroot).rstrip(sep) + sep + destroot = normalize_path(destroot).rstrip(sep) + sep + + # this is supposed to merge a list of files. There will be 2 forms of argument passing. + if isinstance(stufftomerge, basestring): + #A directory is specified. Figure out protection paths, listdir() it and process it. + mergelist = listdir(join(srcroot, stufftomerge)) + offset = stufftomerge + else: + mergelist = stufftomerge + offset = "" + for x in mergelist: + mysrc = join(srcroot, offset, x) + mydest = join(destroot, offset, x) + # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") + myrealdest = join(sep, offset, x) + # stat file once, test using S_* macros many times (faster that way) + try: + mystat = os.lstat(mysrc) + except OSError, e: + writemsg("\n") + writemsg(red("!!! ERROR: There appears to be ")+bold("FILE SYSTEM CORRUPTION.")+red(" A file that is listed\n")) + writemsg(red("!!! as existing is not capable of being stat'd. If you are using an\n")) + writemsg(red("!!! experimental kernel, please boot into a stable one, force an fsck,\n")) + writemsg(red("!!! and ensure your filesystem is in a sane state. ")+bold("'shutdown -Fr now'\n")) + writemsg(red("!!! File: ")+str(mysrc)+"\n", noiselevel=-1) + writemsg(red("!!! Error: ")+str(e)+"\n", noiselevel=-1) + sys.exit(1) + except Exception, e: + writemsg("\n") + writemsg(red("!!! ERROR: An unknown error has occurred during the merge process.\n")) + writemsg(red("!!! A stat call returned the following error for the following file:")) + writemsg( "!!! Please ensure that your filesystem is intact, otherwise report\n") + writemsg( "!!! this as a portage bug at bugs.gentoo.org. Append 'emerge info'.\n") + writemsg( "!!! File: "+str(mysrc)+"\n", noiselevel=-1) + writemsg( "!!! Error: "+str(e)+"\n", noiselevel=-1) + sys.exit(1) + + + mymode = mystat[stat.ST_MODE] + # handy variables; mydest is the target object on the live filesystems; + # mysrc is the source object in the temporary install dir + try: + mydmode = os.lstat(mydest).st_mode + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + #dest file doesn't exist + mydmode = None + + if stat.S_ISLNK(mymode): + # we are merging a symbolic link + myabsto = abssymlink(mysrc) + if myabsto.startswith(srcroot): + myabsto = myabsto[len(srcroot):] + myabsto = myabsto.lstrip(sep) + myto = os.readlink(mysrc) + if self.settings and self.settings["D"]: + if myto.startswith(self.settings["D"]): + myto = myto[len(self.settings["D"]):] + # myrealto contains the path of the real file to which this symlink points. + # we can simply test for existence of this file to see if the target has been merged yet + myrealto = normalize_path(os.path.join(destroot, myabsto)) + if mydmode!=None: + #destination exists + if not stat.S_ISLNK(mydmode): + if stat.S_ISDIR(mydmode): + # directory in the way: we can't merge a symlink over a directory + # we won't merge this, continue with next file... + continue + + if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]): + # Kill file blocking installation of symlink to dir #71787 + pass + elif self.isprotected(mydest): + # Use md5 of the target in ${D} if it exists... + try: + newmd5 = perform_md5(join(srcroot, myabsto)) + except FileNotFound: + # Maybe the target is merged already. + try: + newmd5 = perform_md5(myrealto) + except FileNotFound: + newmd5 = None + mydest = new_protect_filename(mydest, newmd5=newmd5) + + # if secondhand is None it means we're operating in "force" mode and should not create a second hand. + if (secondhand != None) and (not os.path.exists(myrealto)): + # either the target directory doesn't exist yet or the target file doesn't exist -- or + # the target is a broken symlink. We will add this file to our "second hand" and merge + # it later. + secondhand.append(mysrc[len(srcroot):]) + continue + # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly + mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) + if mymtime != None: + writemsg_stdout(">>> %s -> %s\n" % (mydest, myto)) + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") + else: + print "!!! Failed to move file." + print "!!!", mydest, "->", myto + sys.exit(1) + elif stat.S_ISDIR(mymode): + # we are merging a directory + if mydmode != None: + # destination exists + + if bsd_chflags: + # Save then clear flags on dest. + dflags = os.lstat(mydest).st_flags + if dflags != 0: + bsd_chflags.lchflags(mydest, 0) + + if not os.access(mydest, os.W_OK): + pkgstuff = pkgsplit(self.pkg) + writemsg("\n!!! Cannot write to '"+mydest+"'.\n", noiselevel=-1) + writemsg("!!! Please check permissions and directories for broken symlinks.\n") + writemsg("!!! You may start the merge process again by using ebuild:\n") + writemsg("!!! ebuild "+self.settings["PORTDIR"]+"/"+self.cat+"/"+pkgstuff[0]+"/"+self.pkg+".ebuild merge\n") + writemsg("!!! And finish by running this: env-update\n\n") + return 1 + + if stat.S_ISLNK(mydmode) or stat.S_ISDIR(mydmode): + # a symlink to an existing directory will work for us; keep it: + writemsg_stdout("--- %s/\n" % mydest) + if bsd_chflags: + bsd_chflags.lchflags(mydest, dflags) + else: + # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. + if movefile(mydest, mydest+".backup", mysettings=self.settings) is None: + sys.exit(1) + print "bak", mydest, mydest+".backup" + #now create our directory + if self.settings.selinux_enabled(): + import selinux + sid = selinux.get_sid(mysrc) + selinux.secure_mkdir(mydest,sid) + else: + os.mkdir(mydest) + if bsd_chflags: + bsd_chflags.lchflags(mydest, dflags) + os.chmod(mydest, mystat[0]) + os.chown(mydest, mystat[4], mystat[5]) + writemsg_stdout(">>> %s/\n" % mydest) + else: + #destination doesn't exist + if self.settings.selinux_enabled(): + import selinux + sid = selinux.get_sid(mysrc) + selinux.secure_mkdir(mydest, sid) + else: + os.mkdir(mydest) + os.chmod(mydest, mystat[0]) + os.chown(mydest, mystat[4], mystat[5]) + writemsg_stdout(">>> %s/\n" % mydest) + outfile.write("dir "+myrealdest+"\n") + # recurse and merge this directory + if self.mergeme(srcroot, destroot, outfile, secondhand, + join(offset, x), cfgfiledict, thismtime): + return 1 + elif stat.S_ISREG(mymode): + # we are merging a regular file + mymd5 = perform_md5(mysrc, calc_prelink=1) + # calculate config file protection stuff + mydestdir = os.path.dirname(mydest) + moveme = 1 + zing = "!!!" + if mydmode != None: + # destination file exists + if stat.S_ISDIR(mydmode): + # install of destination is blocked by an existing directory with the same name + moveme = 0 + writemsg_stdout("!!! %s\n" % mydest) + elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): + cfgprot = 0 + # install of destination is blocked by an existing regular file, + # or by a symlink to an existing regular file; + # now, config file management may come into play. + # we only need to tweak mydest if cfg file management is in play. + if self.isprotected(mydest): + # we have a protection path; enable config file management. + destmd5 = perform_md5(mydest, calc_prelink=1) + if mymd5 == destmd5: + #file already in place; simply update mtimes of destination + os.utime(mydest, (thismtime, thismtime)) + zing = "---" + moveme = 0 + else: + if mymd5 == cfgfiledict.get(myrealdest, [None])[0]: + """ An identical update has previously been + merged. Skip it unless the user has chosen + --noconfmem.""" + zing = "-o-" + moveme = cfgfiledict["IGNORE"] + cfgprot = cfgfiledict["IGNORE"] + else: + moveme = 1 + cfgprot = 1 + if moveme: + # Merging a new file, so update confmem. + cfgfiledict[myrealdest] = [mymd5] + elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]: + """A previously remembered update has been + accepted, so it is removed from confmem.""" + del cfgfiledict[myrealdest] + if cfgprot: + mydest = new_protect_filename(mydest, newmd5=mymd5) + + # whether config protection or not, we merge the new file the + # same way. Unless moveme=0 (blocking directory) + if moveme: + mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) + if mymtime is None: + sys.exit(1) + zing = ">>>" + else: + mymtime = thismtime + # We need to touch the destination so that on --update the + # old package won't yank the file with it. (non-cfgprot related) + os.utime(mydest, (thismtime,thismtime)) + zing = "---" + if self.settings["USERLAND"] == "Darwin" and myrealdest[-2:] == ".a": + + # XXX kludge, can be killed when portage stops relying on + # md5+mtime, and uses refcounts + # alright, we've fooled w/ mtime on the file; this pisses off static archives + # basically internal mtime != file's mtime, so the linker (falsely) thinks + # the archive is stale, and needs to have it's toc rebuilt. + + myf = open(mydest, "r+") + + # ar mtime field is digits padded with spaces, 12 bytes. + lms = str(thismtime+5).ljust(12) + myf.seek(0) + magic = myf.read(8) + if magic != "!\n": + # not an archive (dolib.a from portage.py makes it here fex) + myf.close() + else: + st = os.stat(mydest) + while myf.tell() < st.st_size - 12: + # skip object name + myf.seek(16, 1) + + # update mtime + myf.write(lms) + + # skip uid/gid/mperm + myf.seek(20, 1) + + # read the archive member's size + x = long(myf.read(10)) + + # skip the trailing newlines, and add the potential + # extra padding byte if it's not an even size + myf.seek(x + 2 + (x % 2),1) + + # and now we're at the end. yay. + myf.close() + mymd5 = perform_md5(mydest, calc_prelink=1) + os.utime(mydest, (thismtime, thismtime)) + + if mymtime != None: + zing = ">>>" + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") + writemsg_stdout("%s %s\n" % (zing,mydest)) + else: + # we are merging a fifo or device node + zing = "!!!" + if mydmode is None: + # destination doesn't exist + if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) != None: + zing = ">>>" + else: + sys.exit(1) + if stat.S_ISFIFO(mymode): + outfile.write("fif %s\n" % myrealdest) + else: + outfile.write("dev %s\n" % myrealdest) + writemsg_stdout(zing + " " + mydest + "\n") + + def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, + mydbapi=None, prev_mtimes=None): + try: + self.lockdb() + return self.treewalk(mergeroot, myroot, inforoot, myebuild, + cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) + finally: + self.unlockdb() + + def getstring(self,name): + "returns contents of a file with whitespace converted to spaces" + if not os.path.exists(self.dbdir+"/"+name): + return "" + myfile = open(self.dbdir+"/"+name,"r") + mydata = myfile.read().split() + myfile.close() + return " ".join(mydata) + + def copyfile(self,fname): + import shutil + shutil.copyfile(fname,self.dbdir+"/"+os.path.basename(fname)) + + def getfile(self,fname): + if not os.path.exists(self.dbdir+"/"+fname): + return "" + myfile = open(self.dbdir+"/"+fname,"r") + mydata = myfile.read() + myfile.close() + return mydata + + def setfile(self,fname,data): + write_atomic(os.path.join(self.dbdir, fname), data) + + def getelements(self,ename): + if not os.path.exists(self.dbdir+"/"+ename): + return [] + myelement = open(self.dbdir+"/"+ename,"r") + mylines = myelement.readlines() + myreturn = [] + for x in mylines: + for y in x[:-1].split(): + myreturn.append(y) + myelement.close() + return myreturn + + def setelements(self,mylist,ename): + myelement = open(self.dbdir+"/"+ename,"w") + for x in mylist: + myelement.write(x+"\n") + myelement.close() + + def isregular(self): + "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" + return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py new file mode 100644 index 000000000..760bf81a0 --- /dev/null +++ b/pym/portage/dbapi/virtual.py @@ -0,0 +1,90 @@ +from portage.dbapi import dbapi + +from portage import cpv_getkey + +class fakedbapi(dbapi): + "This is a dbapi to use for the emptytree function. It's empty, but things can be added to it." + def __init__(self, settings=None): + self.cpvdict = {} + self.cpdict = {} + if settings is None: + from portage import settings + self.settings = settings + self._match_cache = {} + + def _clear_cache(self): + if self._match_cache: + self._match_cache = {} + + def match(self, origdep, use_cache=1): + result = self._match_cache.get(origdep, None) + if result is not None: + return result[:] + result = dbapi.match(self, origdep, use_cache=use_cache) + self._match_cache[origdep] = result + return result[:] + + def cpv_exists(self, mycpv): + return self.cpvdict.has_key(mycpv) + + def cp_list(self, mycp, use_cache=1): + if not self.cpdict.has_key(mycp): + return [] + else: + return self.cpdict[mycp] + + def cp_all(self): + returnme=[] + for x in self.cpdict.keys(): + returnme.extend(self.cpdict[x]) + return returnme + + def cpv_all(self): + return self.cpvdict.keys() + + def cpv_inject(self, mycpv, metadata=None): + """Adds a cpv from the list of available packages.""" + self._clear_cache() + mycp = cpv_getkey(mycpv) + self.cpvdict[mycpv] = metadata + myslot = None + if metadata: + myslot = metadata.get("SLOT", None) + if myslot and mycp in self.cpdict: + # If necessary, remove another package in the same SLOT. + for cpv in self.cpdict[mycp]: + if mycpv != cpv: + other_metadata = self.cpvdict[cpv] + if other_metadata: + if myslot == other_metadata.get("SLOT", None): + self.cpv_remove(cpv) + break + if mycp not in self.cpdict: + self.cpdict[mycp] = [] + if not mycpv in self.cpdict[mycp]: + self.cpdict[mycp].append(mycpv) + + def cpv_remove(self,mycpv): + """Removes a cpv from the list of available packages.""" + self._clear_cache() + mycp = cpv_getkey(mycpv) + if self.cpvdict.has_key(mycpv): + del self.cpvdict[mycpv] + if not self.cpdict.has_key(mycp): + return + while mycpv in self.cpdict[mycp]: + del self.cpdict[mycp][self.cpdict[mycp].index(mycpv)] + if not len(self.cpdict[mycp]): + del self.cpdict[mycp] + + def aux_get(self, mycpv, wants): + if not self.cpv_exists(mycpv): + raise KeyError(mycpv) + metadata = self.cpvdict[mycpv] + if not metadata: + return ["" for x in wants] + return [metadata.get(x, "") for x in wants] + + def aux_update(self, cpv, values): + self._clear_cache() + self.cpvdict[cpv].update(values) diff --git a/pym/portage/dep.py b/pym/portage/dep.py index fb11248ec..1e95fd1da 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -24,6 +24,25 @@ from portage.exception import InvalidData, InvalidAtom from portage.versions import catpkgsplit, catsplit, pkgcmp, pkgsplit, ververify def cpvequal(cpv1, cpv2): + """ + + @param cpv1: CategoryPackageVersion (no operators) Example: "sys-apps/portage-2.1" + @type cpv1: String + @param cpv2: CategoryPackageVersion (no operators) Example: "sys-apps/portage-2.1" + @type cpv2: String + @rtype: Boolean + @returns: + 1. True if cpv1 = cpv2 + 2. False Otherwise + 3. Throws PortageException if cpv1 or cpv2 is not a CPV + + Example Usage: + >>> from portage.dep import cpvequal + >>> cpvequal("sys-apps/portage-2.1","sys-apps/portage-2.1") + >>> True + + """ + split1 = catpkgsplit(cpv1) split2 = catpkgsplit(cpv2) @@ -44,6 +63,7 @@ def strip_empty(myarr): @rtype: Array @return: The array with empty elements removed """ + for x in range(len(myarr)-1, -1, -1): if not myarr[x]: del myarr[x] diff --git a/pym/portage/elog/__init__.py b/pym/portage/elog/__init__.py new file mode 100644 index 000000000..5ba49793d --- /dev/null +++ b/pym/portage/elog/__init__.py @@ -0,0 +1,111 @@ +from portage.const import EBUILD_PHASES +from portage.exception import PortageException +from portage.process import atexit_register +from portage.util import writemsg + +from portage import listdir + +import os + +_elog_atexit_handlers = [] +def elog_process(cpv, mysettings): + mylogfiles = listdir(mysettings["T"]+"/logging/") + # shortcut for packages without any messages + if len(mylogfiles) == 0: + return + # exploit listdir() file order so we process log entries in chronological order + mylogfiles.reverse() + all_logentries = {} + for f in mylogfiles: + msgfunction, msgtype = f.split(".") + if msgfunction not in EBUILD_PHASES: + writemsg("!!! can't process invalid log file: %s\n" % f, + noiselevel=-1) + continue + if not msgfunction in all_logentries: + all_logentries[msgfunction] = [] + msgcontent = open(mysettings["T"]+"/logging/"+f, "r").readlines() + all_logentries[msgfunction].append((msgtype, msgcontent)) + + def filter_loglevels(logentries, loglevels): + # remove unwanted entries from all logentries + rValue = {} + loglevels = map(str.upper, loglevels) + for phase in logentries.keys(): + for msgtype, msgcontent in logentries[phase]: + if msgtype.upper() in loglevels or "*" in loglevels: + if not rValue.has_key(phase): + rValue[phase] = [] + rValue[phase].append((msgtype, msgcontent)) + return rValue + + my_elog_classes = set(mysettings.get("PORTAGE_ELOG_CLASSES", "").split()) + default_logentries = filter_loglevels(all_logentries, my_elog_classes) + + # in case the filters matched all messages and no module overrides exist + if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")): + return + + def combine_logentries(logentries): + # generate a single string with all log messages + rValue = "" + for phase in EBUILD_PHASES: + if not phase in logentries: + continue + for msgtype, msgcontent in logentries[phase]: + rValue += "%s: %s\n" % (msgtype, phase) + for line in msgcontent: + rValue += line + rValue += "\n" + return rValue + + default_fulllog = combine_logentries(default_logentries) + + # pass the processing to the individual modules + logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split() + for s in logsystems: + # allow per module overrides of PORTAGE_ELOG_CLASSES + if ":" in s: + s, levels = s.split(":", 1) + levels = levels.split(",") + mod_logentries = filter_loglevels(all_logentries, levels) + mod_fulllog = combine_logentries(mod_logentries) + else: + mod_logentries = default_logentries + mod_fulllog = default_fulllog + if len(mod_logentries) == 0: + continue + # - is nicer than _ for module names, so allow people to use it. + s = s.replace("-", "_") + try: + # FIXME: ugly ad.hoc import code + # TODO: implement a common portage module loader + logmodule = __import__("portage.elog.mod_"+s) + m = getattr(logmodule, "mod_"+s) + def timeout_handler(signum, frame): + raise PortageException("Timeout in elog_process for system '%s'" % s) + import signal + signal.signal(signal.SIGALRM, timeout_handler) + # Timeout after one minute (in case something like the mail + # module gets hung). + signal.alarm(60) + try: + m.process(mysettings, cpv, mod_logentries, mod_fulllog) + finally: + signal.alarm(0) + if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers: + _elog_atexit_handlers.append(m.finalize) + atexit_register(m.finalize, mysettings) + except (ImportError, AttributeError), e: + writemsg("!!! Error while importing logging modules " + \ + "while loading \"mod_%s\":\n" % str(s)) + writemsg("%s\n" % str(e), noiselevel=-1) + except portage.exception.PortageException, e: + writemsg("%s\n" % str(e), noiselevel=-1) + + # clean logfiles to avoid repetitions + for f in mylogfiles: + try: + os.unlink(os.path.join(mysettings["T"], "logging", f)) + except OSError: + pass diff --git a/pym/portage/elog_modules/mod_custom.py b/pym/portage/elog/mod_custom.py similarity index 100% rename from pym/portage/elog_modules/mod_custom.py rename to pym/portage/elog/mod_custom.py diff --git a/pym/portage/elog_modules/mod_mail.py b/pym/portage/elog/mod_mail.py similarity index 93% rename from pym/portage/elog_modules/mod_mail.py rename to pym/portage/elog/mod_mail.py index bf66d63e4..b84555e8c 100644 --- a/pym/portage/elog_modules/mod_mail.py +++ b/pym/portage/elog/mod_mail.py @@ -1,7 +1,7 @@ # portage.py -- core Portage functionality # Copyright 1998-2004 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Id: mod_mail.py 5781 2007-01-25 16:25:02Z genone $ +# $Id$ import portage.mail, socket diff --git a/pym/portage/elog_modules/mod_mail_summary.py b/pym/portage/elog/mod_mail_summary.py similarity index 100% rename from pym/portage/elog_modules/mod_mail_summary.py rename to pym/portage/elog/mod_mail_summary.py diff --git a/pym/portage/elog_modules/mod_save.py b/pym/portage/elog/mod_save.py similarity index 96% rename from pym/portage/elog_modules/mod_save.py rename to pym/portage/elog/mod_save.py index 614ff81b1..2993b45db 100644 --- a/pym/portage/elog_modules/mod_save.py +++ b/pym/portage/elog/mod_save.py @@ -11,7 +11,7 @@ def process(mysettings, cpv, logentries, fulltext): if not os.path.exists(elogdir): os.makedirs(elogdir) os.chown(elogdir, portage_uid, portage_gid) - os.chmod(elogdir, 0770) + os.chmod(elogdir, 02770) elogfilename = elogdir+"/"+cpv_path+":"+time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time()))+".log" elogfile = open(elogfilename, "w") diff --git a/pym/portage/elog_modules/mod_save_summary.py b/pym/portage/elog/mod_save_summary.py similarity index 100% rename from pym/portage/elog_modules/mod_save_summary.py rename to pym/portage/elog/mod_save_summary.py diff --git a/pym/portage/elog_modules/mod_syslog.py b/pym/portage/elog/mod_syslog.py similarity index 100% rename from pym/portage/elog_modules/mod_syslog.py rename to pym/portage/elog/mod_syslog.py diff --git a/pym/portage/elog_modules/__init__.py b/pym/portage/elog_modules/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pym/portage/output.py b/pym/portage/output.py index bae53ff59..10038efa1 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -174,7 +174,7 @@ def nc_len(mystr): return len(tmp) def xtermTitle(mystr, raw=False): - if havecolor and dotitles and os.environ.has_key("TERM") and sys.stderr.isatty(): + if dotitles and "TERM" in os.environ and sys.stderr.isatty(): myt=os.environ["TERM"] legal_terms = ["xterm","Eterm","aterm","rxvt","screen","kterm","rxvt-unicode","gnome"] for term in legal_terms: -- 2.26.2