From: Fabian Groffen Date: Thu, 16 Nov 2006 19:55:52 +0000 (-0000) Subject: Merged from trunk 5027:5057 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=e8d118061e7455c3e4197bbbc15cbbfa6858ed8a;p=portage.git Merged from trunk 5027:5057 For bug #154989, use a single lock on /var/db/pkg since finer grained locking is unnecessary and often insufficient. For bug #154989, lock /var/db/pkg while determining which packages to unmerge, then release the lock. The lock will be reacquired/released for each individual unmerge, allowing opportunities for other processes to acquire it. Use try/finally to ensure that locks are properly released in portdbapi.aux_get(). Use try/finally to ensure that locks are properly released in perform_checksum(). Use try/finally to ensure that locks are properly released in emergelog(). Use try/finally to ensure that locks are properly released in dblink.unmerge(). Remove reentrance support for dblink.lockdb() since it's no longer used and it won't necessarily work correctly now that a global vdb lock is used. For bug #154986, show the usual "Filesize does not match recorded size" message even for fetch restricted files. For bug #154897, show a summary of fetch restrictions if there are any. Cache depgraph.altlist results, since altlist is called multiple times and can be time consuming for a large depgraph. Make fakedbapi support generic metadata instead of just slots. Remove useless unmerge() call after collision-protect has prevented anything from being merged. Fix a lock file leak. Make portage.global_updates() return any updates that may have been performed. Only reload the config if global updates have actually been performed. Automatically convert --ask to --pretend when the user doesn't have the privileges for --ask. Thanks to Daniel Barkalow for the initial patch (submitted on gentoo-portage mailing list). Don't try to lock the vdb unless privileges are sufficient. For bug #141118, make dep_check do recursive expansion of new-style virtuals. Add a note about removal of new-style virtuals in the depclean output. svn path=/main/branches/prefix/; revision=5059 --- diff --git a/bin/emerge b/bin/emerge index 25cb25f3a..09456bbf6 100755 --- a/bin/emerge +++ b/bin/emerge @@ -243,15 +243,18 @@ def emergelog(xterm_titles, mystr, short_msg=None): portage_util.apply_secpass_permissions(file_path, uid=portage.portage_uid, gid=portage.portage_gid, mode=0660) - - l=portage_locks.lockfile(mylogfile) - # seek because we may have gotten held up by the lock. - # if so, we may not be positioned at the end of the file. - mylogfile.seek(0,2) - mylogfile.write(str(time.time())[:10]+": "+mystr+"\n") - mylogfile.flush() - portage_locks.unlockfile(l) - mylogfile.close() + mylock = None + try: + mylock = portage_locks.lockfile(mylogfile) + # seek because we may have gotten held up by the lock. + # if so, we may not be positioned at the end of the file. + mylogfile.seek(0, 2) + mylogfile.write(str(time.time())[:10]+": "+mystr+"\n") + mylogfile.flush() + finally: + if mylock: + portage_locks.unlockfile(mylock) + mylogfile.close() except (IOError,OSError,portage_exception.PortageException), e: if secpass >= 1: print >> sys.stderr, "emergelog():",e @@ -743,7 +746,7 @@ class depgraph: if "empty" not in self.myparams or self.target_root != "/": for pkg in vardb.cpv_all(): myslot = vardb.aux_get(pkg, ["SLOT"])[0] - fakedb.cpv_inject(pkg, myslot=myslot) + fakedb.cpv_inject(pkg, metadata={"SLOT":myslot}) if self.target_root != "/": self.mydbapi[self.target_root] = \ portage.fakedbapi(settings=settings) @@ -752,7 +755,7 @@ class depgraph: if "empty" not in self.myparams: for pkg in vardb.cpv_all(): myslot = vardb.aux_get(pkg, ["SLOT"])[0] - fakedb.cpv_inject(pkg, myslot=myslot) + fakedb.cpv_inject(pkg, metadata={"SLOT":myslot}) if "--usepkg" in self.myopts: trees["/"]["bintree"].populate( "--getbinpkg" in self.myopts, "--getbinpkgonly" in self.myopts) @@ -764,6 +767,7 @@ class depgraph: self.global_updates = {} self.blocker_digraph = digraph() self.blocker_parents = {} + self._altlist_cache = {} def create(self, mybigkey, myparent=None, addme=1, myuse=None, priority=DepPriority(), rev_dep=False, arg=None): @@ -884,7 +888,7 @@ class depgraph: if addme: myslot = mydbapi.aux_get(mykey, ["SLOT"])[0] - self.mydbapi[myroot].cpv_inject(mykey, myslot=myslot) + self.mydbapi[myroot].cpv_inject(mykey, metadata={"SLOT":myslot}) """ This section determines whether we go deeper into dependencies or not. We want to go deeper on a few occasions: @@ -1510,6 +1514,8 @@ class depgraph: self.altlist() def altlist(self, reversed=False): + if reversed in self._altlist_cache: + return self._altlist_cache[reversed][:] mygraph=self.digraph.copy() myblockers = self.blocker_digraph.copy() retlist=[] @@ -1618,7 +1624,7 @@ class depgraph: """ Add any unresolved blocks so that they can be displayed.""" for blocker in self.blocker_parents: retlist.append(blocker.split()) - + self._altlist_cache[reversed] = retlist[:] return retlist def xcreate(self,mode="system"): @@ -1920,9 +1926,11 @@ class depgraph: "fetch" in portdb.aux_get( x[2], ["RESTRICT"])[0].split(): fetch = red("F") + counters.restrict_fetch += 1 if portdb.fetch_check( pkg_key, self.useFlags[myroot][pkg_key]): fetch = green("f") + counters.restrict_fetch_satisfied += 1 #we need to use "--emptrytree" testing here rather than "empty" param testing because "empty" #param is used for -u, where you still *do* want to see when something is being upgraded. @@ -2246,6 +2254,8 @@ class PackageCounters(object): self.reinst = 0 self.blocks = 0 self.totalsize = 0 + self.restrict_fetch = 0 + self.restrict_fetch_satisfied = 0 def __str__(self): total_installs = self.upgrades + self.downgrades + self.newslot + self.new + self.reinst @@ -2282,6 +2292,14 @@ class PackageCounters(object): if total_installs != 0: myoutput.append(")") myoutput.append(", Size of downloads: %s" % format_size(self.totalsize)) + if self.restrict_fetch: + myoutput.append("\nFetch Restriction: %s package" % \ + self.restrict_fetch) + if self.restrict_fetch > 1: + myoutput.append("s") + if self.restrict_fetch_satisfied < self.restrict_fetch: + myoutput.append(bad(" (%s unsatisfied)") % \ + (self.restrict_fetch - self.restrict_fetch_satisfied)) return "".join(myoutput) class MergeTask(object): @@ -2688,172 +2706,197 @@ def unmerge(settings, myopts, vartree, unmerge_action, unmerge_files, global_unmerge=0 xterm_titles = "notitles" not in settings.features - realsyslist = getlist(settings, "system") - syslist = [] - for x in realsyslist: - mycp = portage.dep_getkey(x) - if mycp in settings.getvirtuals(): - providers = [] - for provider in settings.getvirtuals()[mycp]: - if vartree.dbapi.match(provider): - providers.append(provider) - if len(providers) == 1: - syslist.extend(providers) - else: - syslist.append(mycp) - - mysettings = portage.config(clone=settings) - - if not unmerge_files or "world" in unmerge_files or "system" in unmerge_files: - if "unmerge"==unmerge_action: - print - print bold("emerge unmerge")+" can only be used with specific package names, not with "+bold("world")+" or" - print bold("system")+" targets." - print - return 0 - else: - global_unmerge=1 - - localtree = vartree - # process all arguments and add all valid db entries to candidate_catpkgs - if global_unmerge: - if not unmerge_files or "world" in unmerge_files: - candidate_catpkgs.extend(localtree.getallnodes()) - elif "system" in unmerge_files: - candidate_catpkgs.extend(getlist(settings, "system")) - else: - #we've got command-line arguments - if not unmerge_files: - print "\nNo packages to unmerge have been provided.\n" - return 0 - for x in unmerge_files: - arg_parts=x.split('/') - if (x[0] not in [".","/"]) and (arg_parts[-1][-7:] != ".ebuild"): - #possible cat/pkg or dep; treat as such - candidate_catpkgs.append(x) - elif unmerge_action in ["prune","clean"]: - print "\n!!! Prune and clean do not accept individual ebuilds as arguments;\n skipping.\n" - continue + vdb_path = os.path.join(settings["ROOT"], portage.VDB_PATH) + vdb_lock = None + try: + if os.access(vdb_path, os.W_OK): + vdb_lock = portage_locks.lockdir(vdb_path) + realsyslist = getlist(settings, "system") + syslist = [] + for x in realsyslist: + mycp = portage.dep_getkey(x) + if mycp in settings.getvirtuals(): + providers = [] + for provider in settings.getvirtuals()[mycp]: + if vartree.dbapi.match(provider): + providers.append(provider) + if len(providers) == 1: + syslist.extend(providers) else: - # it appears that the user is specifying an installed ebuild and we're in "unmerge" mode, so it's - # ok. - if not os.path.exists(x): - print "\n!!! The path '"+x+"' doesn't exist.\n" - return 0 - - absx = os.path.abspath(x) - sp_absx = absx.split("/") - if sp_absx[-1][-7:] == ".ebuild": - del sp_absx[-1] - absx = string.join(sp_absx,"/") - - sp_absx_len = len(sp_absx) - - vdb_path = os.path.join(settings["ROOT"], portage.VDB_PATH) - vdb_len = len(vdb_path) - - sp_vdb = vdb_path.split("/") - sp_vdb_len = len(sp_vdb) - - if not os.path.exists(absx+"/CONTENTS"): - print "!!! Not a valid db dir: "+str(absx) - return 0 - - if sp_absx_len <= sp_vdb_len: - # The Path is shorter... so it can't be inside the vdb. - print sp_absx - print absx - print "\n!!!",x,"cannot be inside "+vdb_path+"; aborting.\n" - return 0 - - for idx in range(0,sp_vdb_len): - if (idx >= sp_absx_len) or (sp_vdb[idx] != sp_absx[idx]): + syslist.append(mycp) + + mysettings = portage.config(clone=settings) + + if not unmerge_files or "world" in unmerge_files or \ + "system" in unmerge_files: + if "unmerge"==unmerge_action: + print + print bold("emerge unmerge") + " can only be used with " + \ + "specific package names, not with "+bold("world")+" or" + print bold("system")+" targets." + print + return 0 + else: + global_unmerge = 1 + + localtree = vartree + # process all arguments and add all + # valid db entries to candidate_catpkgs + if global_unmerge: + if not unmerge_files or "world" in unmerge_files: + candidate_catpkgs.extend(vartree.dbapi.cpv_all()) + elif "system" in unmerge_files: + candidate_catpkgs.extend(getlist(settings, "system")) + else: + #we've got command-line arguments + if not unmerge_files: + print "\nNo packages to unmerge have been provided.\n" + return 0 + for x in unmerge_files: + arg_parts = x.split('/') + if x[0] not in [".","/"] and \ + arg_parts[-1][-7:] != ".ebuild": + #possible cat/pkg or dep; treat as such + candidate_catpkgs.append(x) + elif unmerge_action in ["prune","clean"]: + print "\n!!! Prune and clean do not accept individual" + \ + " ebuilds as arguments;\n skipping.\n" + continue + else: + # it appears that the user is specifying an installed + # ebuild and we're in "unmerge" mode, so it's ok. + if not os.path.exists(x): + print "\n!!! The path '"+x+"' doesn't exist.\n" + return 0 + + absx = os.path.abspath(x) + sp_absx = absx.split("/") + if sp_absx[-1][-7:] == ".ebuild": + del sp_absx[-1] + absx = "/".join(sp_absx) + + sp_absx_len = len(sp_absx) + + vdb_path = os.path.join(settings["ROOT"], portage.VDB_PATH) + vdb_len = len(vdb_path) + + sp_vdb = vdb_path.split("/") + sp_vdb_len = len(sp_vdb) + + if not os.path.exists(absx+"/CONTENTS"): + print "!!! Not a valid db dir: "+str(absx) + return 0 + + if sp_absx_len <= sp_vdb_len: + # The Path is shorter... so it can't be inside the vdb. print sp_absx print absx - print "\n!!!",x,"is not inside "+vdb_path+"; aborting.\n" + print "\n!!!",x,"cannot be inside "+ \ + vdb_path+"; aborting.\n" return 0 - - print "="+string.join(sp_absx[sp_vdb_len:],"/") - candidate_catpkgs.append("="+string.join(sp_absx[sp_vdb_len:],"/")) - - newline="" - if (not "--quiet" in myopts): - newline="\n" - if settings["ROOT"].startswith(portage_const.EPREFIX) and settings["ROOT"] != portage_const.EPREFIX: - print darkgreen(newline+">>> Using system located in ROOT tree "+settings["ROOT"]) - if (("--pretend" in myopts) or ("--ask" in myopts)) and not ("--quiet" in myopts): - print darkgreen(newline+">>> These are the packages that would be unmerged:") - - pkgmap={} - numselected=0 - for x in candidate_catpkgs: - #cycle through all our candidate deps and determine what will and will not get unmerged - try: - mymatch=localtree.dep_match(x) - except KeyError: - mymatch=None - except ValueError, errpkgs: - print "\n\n!!! The short ebuild name \"" + x + "\" is ambiguous. Please specify" - print "!!! one of the following fully-qualified ebuild names instead:\n" - for i in errpkgs[0]: - print " " + green(i) - print - sys.exit(1) - - if not mymatch and x[0] not in "<>=~": - #add a "=" if missing - mymatch=localtree.dep_match("="+x) - if not mymatch: - if raise_on_missing: - raise portage_exception.PackageNotFound(x) - else: - portage.writemsg("\n--- Couldn't find '%s' to %s.\n" % \ - (x, unmerge_action), noiselevel=-1) - continue - mykey = portage.key_expand( - portage.dep_getkey( - mymatch[0]), mydb=vartree.dbapi, settings=settings) - if not pkgmap.has_key(mykey): - pkgmap[mykey]={"protected":[], "selected":[], "omitted":[] } - if unmerge_action=="unmerge": - for y in mymatch: - if y not in pkgmap[mykey]["selected"]: - pkgmap[mykey]["selected"].append(y) - numselected=numselected+len(mymatch) - - else: - #unmerge_action in ["prune", clean"] - slotmap={} - for mypkg in mymatch: - if unmerge_action=="clean": - myslot=localtree.getslot(mypkg) + + for idx in range(0,sp_vdb_len): + if idx >= sp_absx_len or sp_vdb[idx] != sp_absx[idx]: + print sp_absx + print absx + print "\n!!!", x, "is not inside "+\ + vdb_path+"; aborting.\n" + return 0 + + print "="+"/".join(sp_absx[sp_vdb_len:]) + candidate_catpkgs.append( + "="+"/".join(sp_absx[sp_vdb_len:])) + + newline="" + if (not "--quiet" in myopts): + newline="\n" + if settings["ROOT"] != "/": + print darkgreen(newline+ \ + ">>> Using system located in ROOT tree "+settings["ROOT"]) + if (("--pretend" in myopts) or ("--ask" in myopts)) and \ + not ("--quiet" in myopts): + print darkgreen(newline+\ + ">>> These are the packages that would be unmerged:") + + pkgmap={} + numselected=0 + for x in candidate_catpkgs: + # cycle through all our candidate deps and determine + # what will and will not get unmerged + try: + mymatch=localtree.dep_match(x) + except KeyError: + mymatch=None + except ValueError, errpkgs: + print "\n\n!!! The short ebuild name \"" + \ + x + "\" is ambiguous. Please specify" + print "!!! one of the following fully-qualified " + \ + "ebuild names instead:\n" + for i in errpkgs[0]: + print " " + green(i) + print + sys.exit(1) + + if not mymatch and x[0] not in "<>=~": + #add a "=" if missing + mymatch=localtree.dep_match("="+x) + if not mymatch: + if raise_on_missing: + raise portage_exception.PackageNotFound(x) else: - #since we're pruning, we don't care about slots and put all the pkgs in together - myslot=0 - if not slotmap.has_key(myslot): - slotmap[myslot]={} - slotmap[myslot][localtree.dbapi.cpv_counter(mypkg)]=mypkg - for myslot in slotmap.keys(): - counterkeys=slotmap[myslot].keys() - counterkeys.sort() - if not counterkeys: + portage.writemsg("\n--- Couldn't find '%s' to %s.\n" % \ + (x, unmerge_action), noiselevel=-1) continue - counterkeys.sort() - pkgmap[mykey]["protected"].append(slotmap[myslot][counterkeys[-1]]) - del counterkeys[-1] - #be pretty and get them in order of merge: - for ckey in counterkeys: - pkgmap[mykey]["selected"].append(slotmap[myslot][ckey]) - numselected=numselected+1 - #ok, now the last-merged package is protected, and the rest are selected - if global_unmerge and not numselected: - print "\n>>> No outdated packages were found on your system.\n" - return 0 - - if not numselected: - print "\n>>> No packages selected for removal by",unmerge_action+".\n" - return 0 - + mykey = portage.key_expand( + portage.dep_getkey( + mymatch[0]), mydb=vartree.dbapi, settings=settings) + if not pkgmap.has_key(mykey): + pkgmap[mykey]={"protected":[], "selected":[], "omitted":[] } + if unmerge_action=="unmerge": + for y in mymatch: + if y not in pkgmap[mykey]["selected"]: + pkgmap[mykey]["selected"].append(y) + numselected=numselected+len(mymatch) + + else: + #unmerge_action in ["prune", clean"] + slotmap={} + for mypkg in mymatch: + if unmerge_action=="clean": + myslot=localtree.getslot(mypkg) + else: + # since we're pruning, we don't care about slots + # and put all the pkgs in together + myslot=0 + if not slotmap.has_key(myslot): + slotmap[myslot]={} + slotmap[myslot][localtree.dbapi.cpv_counter(mypkg)]=mypkg + for myslot in slotmap.keys(): + counterkeys=slotmap[myslot].keys() + counterkeys.sort() + if not counterkeys: + continue + counterkeys.sort() + pkgmap[mykey]["protected"].append( + slotmap[myslot][counterkeys[-1]]) + del counterkeys[-1] + #be pretty and get them in order of merge: + for ckey in counterkeys: + pkgmap[mykey]["selected"].append(slotmap[myslot][ckey]) + numselected=numselected+1 + # ok, now the last-merged package + # is protected, and the rest are selected + if global_unmerge and not numselected: + print "\n>>> No outdated packages were found on your system.\n" + return 0 + + if not numselected: + print "\n>>> No packages selected for removal by", \ + unmerge_action+".\n" + return 0 + finally: + if vdb_lock: + portage_locks.unlockdir(vdb_lock) for x in pkgmap.keys(): for y in localtree.dep_match(x): if y not in pkgmap[x]["omitted"] and \ @@ -3008,10 +3051,6 @@ def post_emerge(settings, mtimedb, retval): chk_updated_info_files( normalize_path(target_root + portage_const.EPREFIX), infodirs, info_mtimes, retval) - - chk_updated_cfg_files( - normalize_path(target_root + portage_const.EPREFIX), - config_protect) mtimedb.commit() sys.exit(retval) @@ -3484,12 +3523,11 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if os.path.exists(myportdir+"/metadata/cache") and updatecache_flg: action_metadata(settings, portdb, myopts) - portage.global_updates( - settings, trees, mtimedb["updates"]) - mtimedb.commit() - # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings["ROOT"]]["porttree"].dbapi + if portage.global_updates(settings, trees, mtimedb["updates"]): + mtimedb.commit() + # Reload the whole config from scratch. + settings, trees, mtimedb = load_emerge_config(trees=trees) + portdb = trees[settings["ROOT"]]["porttree"].dbapi mybestpv = portdb.xmatch("bestmatch-visible", "sys-apps/portage") mypvs = portage.best( @@ -3856,7 +3894,9 @@ def action_depclean(settings, trees, ldpath_mtimes, print warn_prefix + "be kept. They can be manually added to this set with" print warn_prefix + good("`emerge --noreplace `") + ". Packages that are listed in" print warn_prefix + "package.provided (see portage(5)) will be removed by" - print warn_prefix + "depclean, even if they are part of the world set." + print warn_prefix + "depclean, even if they are part of the world set. New-style" + print warn_prefix + "virtuals (virtual/* packages installed by older versions of" + print warn_prefix + "portage) will also be removed." print warn_prefix print warn_prefix + "As a safety measure, depclean will not remove any packages" print warn_prefix + "unless *all* required dependencies have been resolved. As a" @@ -4388,11 +4428,11 @@ def emerge_main(): os.umask(022) settings, trees, mtimedb = load_emerge_config() portdb = trees[settings["ROOT"]]["porttree"].dbapi - portage.global_updates(settings, trees, mtimedb["updates"]) - mtimedb.commit() - # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings["ROOT"]]["porttree"].dbapi + if portage.global_updates(settings, trees, mtimedb["updates"]): + mtimedb.commit() + # Reload the whole config from scratch. + settings, trees, mtimedb = load_emerge_config(trees=trees) + portdb = trees[settings["ROOT"]]["porttree"].dbapi ldpath_mtimes = mtimedb["ldpath"] xterm_titles = "notitles" not in settings.features @@ -4569,15 +4609,18 @@ def emerge_main(): # We've already allowed "--version" and "--help" above. if "--pretend" not in myopts and \ myaction not in ("search","info"): - if portage.secpass >= 1: - if "--fetchonly" not in myopts and \ - "--fetch-all-uri" not in myopts and \ - myaction not in ("metadata", "regen"): + if portage.secpass < 1 or \ + not ("--fetchonly" in myopts or \ + "--fetch-all-uri" in myopts or \ + myaction in ("metadata", "regen")): + if "--ask" in myopts: + myopts["--pretend"] = True + del myopts["--ask"] + print "root access would be required..." + \ + " adding --pretend to options." + else: print "emerge: root access required." sys.exit(1) - else: - print "emerge: root access required." - sys.exit(1) disable_emergelog = False for x in ("--pretend", "--fetchonly", "--fetch-all-uri"): diff --git a/pym/portage.py b/pym/portage.py index aa21cc37c..c7367cc6d 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -1398,8 +1398,7 @@ class config: for mypath, (gid, mode, modemask) in dir_mode_map.iteritems(): try: - mydir = normalize_path( - os.path.join(self["ROOT"] + EPREFIX, mypath)) + mydir = normalize_path(os.path.join(self["ROOT"] + EPREFIX, mypath)) portage_util.ensure_dirs(mydir, gid=gid, mode=mode, mask=modemask) except portage_exception.PortageException, e: writemsg("!!! Directory initialization failed: '%s'\n" % mydir, @@ -2270,7 +2269,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", # assume that it is fully downloaded. continue else: - if mystat.st_size < mydigests[myfile]["size"]: + if mystat.st_size < mydigests[myfile]["size"] and \ + not restrict_fetch: fetched = 1 # Try to resume this download. else: verified_ok, reason = portage_checksum.verify_all( @@ -2283,7 +2283,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", writemsg(("!!! Got: %s\n" + \ "!!! Expected: %s\n") % \ (reason[1], reason[2]), noiselevel=-1) - if can_fetch: + if can_fetch and not restrict_fetch: writemsg("Refetching...\n\n", noiselevel=-1) os.unlink(myfile_path) @@ -3513,6 +3513,77 @@ def dep_virtual(mysplit, mysettings): newsplit.append(x) return newsplit +def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", + trees=None, **kwargs): + """Recursively expand new-style virtuals so as to collapse one or more + levels of indirection. The new-style virtuals should never be installed + themselves. They are only used to expand virtual dependencies. Virtual + blockers are supported but only when the virtual expands to a single + atom because it wouldn't necessarily make sense to block all the components + of a compound virtual. When more than one new-style virtual is matched, + the matches are sorted from highest to lowest versions and the atom is + expanded to || ( highest match ... lowest match ).""" + newsplit = [] + # According to GLEP 37, RDEPEND is the only dependency type that is valid + # for new-style virtuals. Repoman should enforce this. + dep_keys = ["RDEPEND", "DEPEND", "PDEPEND"] + def compare_pkgs(a, b): + return pkgcmp(b[1], a[1]) + portdb = trees[myroot]["porttree"].dbapi + for x in mysplit: + if x == "||": + newsplit.append(x) + continue + elif isinstance(x, list): + newsplit.append(_expand_new_virtuals(x, edebug, mydbapi, + mysettings, myroot=myroot, trees=trees, **kwargs)) + continue + elif not dep_getkey(x).startswith("virtual/"): + newsplit.append(x) + continue + isblocker = x.startswith("!") + match_atom = x + if isblocker: + match_atom = x[1:] + pkgs = [] + for cpv in portdb.match(match_atom): + # only use new-style matches + if cpv.startswith("virtual/"): + pkgs.append((cpv, pkgsplit(cpv))) + if not pkgs: + newsplit.append(x) + continue + pkgs.sort(compare_pkgs) # Prefer higher versions. + if isblocker: + a = [] + else: + a = ['||'] + for y in pkgs: + depstring = " ".join(portdb.aux_get(y[0], dep_keys)) + if edebug: + print "Virtual Parent: ", y[0] + print "Virtual Depstring:", depstring + mycheck = dep_check(depstring, mydbapi, mysettings, myroot=myroot, + trees=trees, **kwargs) + if not mycheck[0]: + raise portage_exception.ParseError( + "%s: %s '%s'" % (y[0], mycheck[1], depstring)) + if isblocker: + virtual_atoms = [atom for atom in mycheck[1] \ + if not atom.startswith("!")] + if len(virtual_atoms) == 1: + # It wouldn't make sense to block all the components of a + # compound virtual, so only a single atom block is allowed. + a.append("!" + virtual_atoms[0]) + else: + a.append(mycheck[1]) + if isblocker and not a: + # Probably a compound virtual. Pass the atom through unprocessed. + newsplit.append(x) + continue + newsplit.append(a) + return newsplit + def dep_eval(deplist): if not deplist: return 1 @@ -3682,7 +3753,7 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, use_cache=1, use_binaries=0, myroot="/", trees=None, str_matches=None, return_all_deps=False): """Takes a depend string and parses the condition.""" - + edebug = mysettings.get("PORTAGE_DEBUG", None) == "1" #check_config_instance(mysettings) if use=="yes": @@ -3736,6 +3807,17 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, elif mysplit==[]: #dependencies were reduced to nothing return [1,[]] + + # Recursively expand new-style virtuals so as to + # collapse one or more levels of indirection. + try: + mysplit = _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, + use=use, mode=mode, myuse=myuse, use_cache=use_cache, + use_binaries=use_binaries, myroot=myroot, trees=trees, + str_matches=str_matches, return_all_deps=return_all_deps) + except portage_exception.ParseError, e: + return [0, str(e)] + mysplit2=mysplit[:] mysplit2 = dep_wordreduce(mysplit2, mysettings, mydbapi, mode, str_matches=str_matches, use_cache=use_cache) @@ -4244,10 +4326,13 @@ class fakedbapi(dbapi): def cpv_all(self): return self.cpvdict.keys() - def cpv_inject(self, mycpv, myslot=None): + def cpv_inject(self, mycpv, metadata=None): """Adds a cpv from the list of available packages.""" mycp=cpv_getkey(mycpv) - self.cpvdict[mycpv] = myslot + self.cpvdict[mycpv] = metadata + myslot = None + if metadata: + myslot = metadata.get("SLOT", None) if myslot and mycp in self.cpdict: # If necessary, remove another package in the same SLOT. for cpv in self.cpdict[mycp]: @@ -4284,13 +4369,10 @@ class fakedbapi(dbapi): def aux_get(self, mycpv, wants): if not self.cpv_exists(mycpv): raise KeyError(mycpv) - values = [] - for x in wants: - if x == "SLOT": - values.append(self.cpvdict[mycpv]) - else: - values.append("") - return values + metadata = self.cpvdict[mycpv] + if not metadata: + return ["" for x in wants] + return [metadata.get(x, "") for x in wants] class bindbapi(fakedbapi): def __init__(self, mybintree=None, settings=None): @@ -4933,7 +5015,6 @@ class portdbapi(dbapi): def __init__(self,porttree_root,mysettings=None): portdbapi.portdbapi_instances.append(self) - self.lock_held = 0; if mysettings: self.mysettings = mysettings @@ -5158,48 +5239,44 @@ class portdbapi(dbapi): else: mydbkey = self.depcachedir+"/aux_db_key_temp" - if self.lock_held: - raise "Lock is already held by me?" - self.lock_held = 1 - mylock = portage_locks.lockfile(mydbkey, wantnewlockfile=1) - - if os.path.exists(mydbkey): + mylock = None + try: + mylock = portage_locks.lockfile(mydbkey, wantnewlockfile=1) try: os.unlink(mydbkey) except (IOError, OSError), e: - portage_locks.unlockfile(mylock) - self.lock_held = 0 - writemsg("Uncaught handled exception: %(exception)s\n" % {"exception":str(e)}) - raise - - self.doebuild_settings.reset() - myret = doebuild(myebuild, "depend", "/", self.doebuild_settings, - dbkey=mydbkey, tree="porttree", mydbapi=self) - if myret: - portage_locks.unlockfile(mylock) - self.lock_held = 0 - #depend returned non-zero exit code... - writemsg(str(red("\naux_get():")+" (0) Error in "+mycpv+" ebuild. ("+str(myret)+")\n" - " Check for syntax error or corruption in the ebuild. (--debug)\n\n"), - noiselevel=-1) - raise KeyError - - try: - mycent=open(mydbkey,"r") - os.unlink(mydbkey) - mylines=mycent.readlines() - mycent.close() + if e.errno != errno.ENOENT: + raise + del e - except (IOError, OSError): - portage_locks.unlockfile(mylock) - self.lock_held = 0 - writemsg(str(red("\naux_get():")+" (1) Error in "+mycpv+" ebuild.\n" - " Check for syntax error or corruption in the ebuild. (--debug)\n\n"), - noiselevel=-1) - raise KeyError + self.doebuild_settings.reset() + myret = doebuild(myebuild, "depend", "/", + self.doebuild_settings, dbkey=mydbkey, tree="porttree", + mydbapi=self) + if myret != os.EX_OK: + #depend returned non-zero exit code... + writemsg((red("\naux_get():") + \ + " (0) Error in '%s'. (%s)\n" + \ + " Check for syntax error or " + \ + "corruption in the ebuild. (--debug)\n\n") % \ + (myebuild, myret), noiselevel=-1) + raise KeyError(mycpv) - portage_locks.unlockfile(mylock) - self.lock_held = 0 + try: + mycent = open(mydbkey, "r") + os.unlink(mydbkey) + mylines = mycent.readlines() + mycent.close() + except (IOError, OSError): + writemsg((red("\naux_get():") + \ + " (1) Error in '%s' ebuild.\n" + \ + " Check for syntax error or " + \ + "corruption in the ebuild. (--debug)\n\n") % myebuild, + noiselevel=-1) + raise KeyError(mycpv) + finally: + if mylock: + portage_locks.unlockfile(mylock) mydata = {} for x in range(0,len(mylines)): @@ -6033,9 +6110,7 @@ class dblink: self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg self.dbdir = self.dbpkgdir - self.lock_pkg = None - self.lock_tmp = None - self.lock_num = 0 # Count of the held locks on the db. + self._lock_vdb = None self.settings = mysettings if self.settings==1: @@ -6051,16 +6126,14 @@ class dblink: self._contents_inodes = None def lockdb(self): - if self.lock_num == 0: - self.lock_pkg = portage_locks.lockdir(self.dbpkgdir) - self.lock_tmp = portage_locks.lockdir(self.dbtmpdir) - self.lock_num += 1 + if self._lock_vdb: + raise AssertionError("Lock already held.") + self._lock_vdb = portage_locks.lockdir(self.dbroot) def unlockdb(self): - self.lock_num -= 1 - if self.lock_num == 0: - portage_locks.unlockdir(self.lock_tmp) - portage_locks.unlockdir(self.lock_pkg) + if self._lock_vdb: + portage_locks.unlockdir(self._lock_vdb) + self._lock_vdb = None def getpath(self): "return path to location of db information (for >>> informational display)" @@ -6398,9 +6471,6 @@ class dblink: if not os.path.exists(self.dbcatdir): os.makedirs(self.dbcatdir) - # This blocks until we can get the dirs to ourselves. - self.lockdb() - otherversions=[] for v in self.vartree.dbapi.cp_list(self.mysplit[0]): otherversions.append(v.split("/")[1]) @@ -6479,12 +6549,6 @@ class dblink: print print red("package "+self.cat+"/"+self.pkg+" NOT merged") print - # Why is the package already merged here db-wise? Shouldn't be the case - # only unmerge if it ia new package and has no contents - if not self.getcontents(): - self.unmerge(ldpath_mtimes=prev_mtimes) - self.delete() - self.unlockdb() print print "Searching all installed packages for file collisions..." print "Press Ctrl-C to Stop" @@ -6632,7 +6696,6 @@ class dblink: self.delete() movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) contents = self.getcontents() - self.unlockdb() #write out our collection of md5sums if cfgfiledict.has_key("IGNORE"): @@ -6966,8 +7029,12 @@ class dblink: def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None): - return self.treewalk(mergeroot, myroot, inforoot, myebuild, - cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) + try: + self.lockdb() + return self.treewalk(mergeroot, myroot, inforoot, myebuild, + cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) + finally: + self.unlockdb() def getstring(self,name): "returns contents of a file with whitespace converted to spaces" @@ -7214,7 +7281,20 @@ def portageexit(): atexit_register(portageexit) def global_updates(mysettings, trees, prev_mtimes): - """Perform new global updates if they exist in $PORTDIR/profiles/updates/.""" + """ + Perform new global updates if they exist in $PORTDIR/profiles/updates/. + + @param mysettings: A config instance for ROOT="/". + @type mysettings: config + @param trees: A dictionary containing portage trees. + @type trees: dict + @param prev_mtimes: A dictionary containing mtimes of files located in + $PORTDIR/profiles/updates/. + @type prev_mtimes: dict + @rtype: None or List + @return: None if no were no updates, otherwise a list of update commands + that have been performed. + """ # only do this if we're root and not running repoman/ebuild digest global secpass if secpass < 2 or "SANDBOX_ACTIVE" in os.environ: @@ -7229,6 +7309,7 @@ def global_updates(mysettings, trees, prev_mtimes): except portage_exception.DirectoryNotFound: writemsg("--- 'profiles/updates' is empty or not available. Empty portage tree?\n") return + myupd = None if len(update_data) > 0: do_upgrade_packagesmessage = 0 myupd = [] @@ -7292,6 +7373,8 @@ def global_updates(mysettings, trees, prev_mtimes): writemsg_stdout(" ** Skipping packages. Run 'fixpackages' or set it in FEATURES to fix the") writemsg_stdout("\n tbz2's in the packages directory. "+bold("Note: This can take a very long time.")) writemsg_stdout("\n") + if myupd: + return myupd #continue setting up other trees diff --git a/pym/portage_checksum.py b/pym/portage_checksum.py index 6ba58e75c..d3659d206 100644 --- a/pym/portage_checksum.py +++ b/pym/portage_checksum.py @@ -166,35 +166,35 @@ def perform_checksum(filename, hashname="MD5", calc_prelink=0): myfilename = filename[:] prelink_tmpfile = os.path.join("/", PRIVATE_PATH, "prelink-checksum.tmp." + str(os.getpid())) mylock = None - - if calc_prelink and prelink_capable: - mylock = portage_locks.lockfile(prelink_tmpfile, wantnewlockfile=1) - # Create non-prelinked temporary file to checksum. - # Files rejected by prelink are summed in place. - retval=portage_exec.spawn([PRELINK_BINARY,"--undo","-o",prelink_tmpfile,filename],fd_pipes={}) - if retval==0: - #portage_util.writemsg(">>> prelink checksum '"+str(filename)+"'.\n") - myfilename=prelink_tmpfile try: - if hashname not in hashfunc_map: - raise portage_exception.DigestException, hashname+" hash function not available (needs dev-python/pycrypto)" - myhash, mysize = hashfunc_map[hashname](myfilename) - except (OSError, IOError), e: - if e.errno == errno.ENOENT: - raise portage_exception.FileNotFound(myfilename) - else: - raise - if calc_prelink and prelink_capable: + if calc_prelink and prelink_capable: + mylock = portage_locks.lockfile(prelink_tmpfile, wantnewlockfile=1) + # Create non-prelinked temporary file to checksum. + # Files rejected by prelink are summed in place. + retval = portage_exec.spawn([PRELINK_BINARY, "--undo", "-o", + prelink_tmpfile, filename], fd_pipes={}) + if retval == os.EX_OK: + myfilename = prelink_tmpfile try: - os.unlink(prelink_tmpfile) - except OSError, oe: - if oe.errno == errno.ENOENT: - pass - else: - raise - portage_locks.unlockfile(mylock) - - return (myhash,mysize) + if hashname not in hashfunc_map: + raise portage_exception.DigestException(hashname + \ + " hash function not available (needs dev-python/pycrypto)") + myhash, mysize = hashfunc_map[hashname](myfilename) + except (OSError, IOError), e: + if e.errno == errno.ENOENT: + raise portage_exception.FileNotFound(myfilename) + raise + if calc_prelink and prelink_capable: + try: + os.unlink(prelink_tmpfile) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + return myhash, mysize + finally: + if mylock: + portage_locks.unlockfile(mylock) def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0): """