From b5ccd3faf70fe14ad18ead5ecca594b1e7714d40 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 3 Mar 2007 11:33:22 +0000 Subject: [PATCH] Merged from trunk 6132:6146 When emerge automatically resumes, add --ignore-default-opts to the command line in order to prevent things like --ask from leaking in via EMERGE_DEFAULT_OPTS. Thanks to Peter Weller for reporting. For bug #169046, fill in the --tree display with "nomerge" nodes so that the user can always see at lease 1 parent on any non-root node. Packages may appear in the display list multiple times, and will be marked "nomerge" when their position in the list does not correspond to actual merge order. Fix types.InstanceType test which is failing now for some reason. remove category regex, dumb idea, sorry I did it (WarnerBro) s/node/selected_parent/ for clarity For bug #138840, show a more informative message when waiting for a distfiles lock due to parallel-fetch. Thanks to David Watzke for the initial patch. Add missing sys import. svn path=/main/branches/prefix/; revision=6150 --- pym/emerge/__init__.py | 118 ++++++++++++++++++++++++++++++---------- pym/portage/__init__.py | 15 ++++- pym/portage/const.py | 2 +- pym/portage/data.py | 17 +++--- pym/portage/locks.py | 13 +++-- pym/portage/versions.py | 6 -- 6 files changed, 118 insertions(+), 53 deletions(-) diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index df930ea11..2125367aa 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -2332,42 +2332,81 @@ class depgraph: for t in self.settings["PORTDIR_OVERLAY"].split()] tree_nodes = [] - node_depth = {} + display_list = [] + mygraph = self._parent_child_digraph i = 0 depth = 0 for x in mylist: if "blocks" == x[0]: + display_list.append((x, 0, True)) continue graph_key = tuple(x) if "--tree" in self.myopts: depth = len(tree_nodes) while depth and graph_key not in \ - self.digraph.child_nodes(tree_nodes[depth-1]): + mygraph.child_nodes(tree_nodes[depth-1]): depth -= 1 - tree_nodes = tree_nodes[:depth] - tree_nodes.append(graph_key) - node_depth[graph_key] = depth + if depth: + tree_nodes = tree_nodes[:depth] + tree_nodes.append(graph_key) + display_list.append((x, depth, True)) + else: + traversed_nodes = set() # prevent endless circles + traversed_nodes.add(graph_key) + def add_parents(current_node, ordered): + parent_nodes = mygraph.parent_nodes(current_node) + if parent_nodes: + child_nodes = set(mygraph.child_nodes(current_node)) + selected_parent = None + # First, try to avoid a direct cycle. + for node in parent_nodes: + if node not in traversed_nodes and \ + node not in child_nodes: + selected_parent = node + break + if not selected_parent: + # A direct cycle is unavoidable. + for node in parent_nodes: + if node not in traversed_nodes: + selected_parent = node + break + if selected_parent: + traversed_nodes.add(selected_parent) + add_parents(selected_parent, False) + display_list.append((list(current_node), + len(tree_nodes), ordered)) + tree_nodes.append(current_node) + tree_nodes = [] + add_parents(graph_key, True) + else: + display_list.append((x, depth, True)) + mylist = display_list last_merge_depth = 0 for i in xrange(len(mylist)-1,-1,-1): - if "blocks" == mylist[i][0]: + graph_key, depth, ordered = mylist[i] + if not ordered and depth == 0 and i > 1 \ + and graph_key == mylist[i-1][0]: + # An ordered node got a consecutive duplicate when the tree was + # being filled in. + del mylist[i] continue - graph_key = tuple(mylist[i]) - if mylist[i][-1] != "nomerge": - last_merge_depth = node_depth[graph_key] + if "blocks" == graph_key[0]: continue - if node_depth[graph_key] >= last_merge_depth or \ + if graph_key[-1] != "nomerge": + last_merge_depth = depth + continue + if depth >= last_merge_depth or \ i < len(mylist) - 1 and \ - node_depth[graph_key] >= node_depth[tuple(mylist[i+1])]: + depth >= mylist[i+1][1]: del mylist[i] - del node_depth[graph_key] - del tree_nodes display_overlays=False # files to fetch list - avoids counting a same file twice # in size display (verbose mode) myfetchlist=[] - for x in mylist: + for mylist_index in xrange(len(mylist)): + x, depth, ordered = mylist[mylist_index] pkg_type = x[0] myroot = x[1] pkg_key = x[2] @@ -2381,7 +2420,8 @@ class depgraph: if x[0]=="blocks": addl=""+red("B")+" "+fetch+" " - counters.blocks += 1 + if ordered: + counters.blocks += 1 resolved = portage.key_expand( pkg_key, mydb=vardb, settings=pkgsettings) if "--columns" in self.myopts and "--quiet" in self.myopts: @@ -2422,11 +2462,13 @@ class depgraph: "fetch" in portdb.aux_get( x[2], ["RESTRICT"])[0].split(): fetch = red("F") - counters.restrict_fetch += 1 + if ordered: + counters.restrict_fetch += 1 if portdb.fetch_check( pkg_key, self.useFlags[myroot][pkg_key]): fetch = green("f") - counters.restrict_fetch_satisfied += 1 + if ordered: + counters.restrict_fetch_satisfied += 1 #we need to use "--emptrytree" testing here rather than "empty" param testing because "empty" #param is used for -u, where you still *do* want to see when something is being upgraded. @@ -2434,7 +2476,8 @@ class depgraph: if vardb.cpv_exists(pkg_key): addl=" "+yellow("R")+fetch+" " if x[3] != "nomerge": - counters.reinst += 1 + if ordered: + counters.reinst += 1 elif vardb.match(portage.dep_getkey(pkg_key)): mynewslot = mydbapi.aux_get(pkg_key, ["SLOT"])[0] myoldlist = self.trees[x[1]]["vartree"].dbapi.match( @@ -2447,15 +2490,18 @@ class depgraph: if portage.pkgcmp(portage.pkgsplit(x[2]), portage.pkgsplit(myoldbest)) < 0: # Downgrade in slot addl+=turquoise("U")+blue("D") - counters.downgrades += 1 + if ordered: + counters.downgrades += 1 else: # Update in slot addl+=turquoise("U")+" " - counters.upgrades += 1 + if ordered: + counters.upgrades += 1 else: # New slot, mark it new. addl=" "+green("NS")+fetch+" " - counters.newslot += 1 + if ordered: + counters.newslot += 1 if "--changelog" in self.myopts: slot_atom = "%s:%s" % (portage.dep_getkey(pkg_key), @@ -2467,7 +2513,8 @@ class depgraph: inst_matches[0], pkg_key)) else: addl=" "+green("N")+" "+fetch+" " - counters.new += 1 + if ordered: + counters.new += 1 verboseadd="" @@ -2571,7 +2618,8 @@ class depgraph: if myfetchfile not in myfetchlist: mysize+=myfilesdict[myfetchfile] myfetchlist.append(myfetchfile) - counters.totalsize += mysize + if ordered: + counters.totalsize += mysize verboseadd+=format_size(mysize)+" " # overlay verbose @@ -2606,7 +2654,7 @@ class depgraph: oldlp=mywidth-30 newlp=oldlp-30 - indent = " " * node_depth[tuple(x)] + indent = " " * depth if myoldbest: myoldbest=portage.pkgsplit(myoldbest)[1]+"-"+portage.pkgsplit(myoldbest)[2] @@ -2633,7 +2681,7 @@ class depgraph: myprint=myprint+myoldbest myprint=myprint+darkgreen("to "+x[1])+" "+verboseadd else: - if x[3] == "nomerge": + if x[-1] == "nomerge" or not ordered: myprint = darkblue("[nomerge ] ") else: myprint = "[" + pkg_type + " " + addl + "] " @@ -2655,7 +2703,7 @@ class depgraph: myprint=myprint+(" "*(oldlp-nc_len(myprint))) myprint=myprint+myoldbest+" "+verboseadd else: - if x[3]=="nomerge": + if x[-1] == "nomerge" or not ordered: myprint=darkblue("[nomerge ] "+indent+x[2]+" "+myoldbest+" ")+verboseadd else: myprint="["+x[0]+" "+addl+"] "+indent+darkgreen(x[2])+" "+myoldbest+" "+verboseadd @@ -2672,7 +2720,7 @@ class depgraph: myversion = "%s-%s" % (mysplit[1], mysplit[2]) if myversion != portage.VERSION and "--quiet" not in self.myopts: - if mylist.index(x) < len(mylist) - 1 and \ + if mylist_index < len(mylist) - 1 and \ "livecvsportage" not in self.settings.features: p.append(colorize("WARN", "*** Portage will stop merging at this point and reload itself,")) p.append(colorize("WARN", " then resume the merge.")) @@ -2927,7 +2975,12 @@ class MergeTask(object): fetch_env["FEATURES"] = fetch_env.get("FEATURES", "") + " -cvs" fetch_env["PORTAGE_NICENESS"] = "0" fetch_args = [sys.argv[0], "--resume", "--fetchonly"] - for myopt, myarg in self.myopts.iteritems(): + resume_opts = self.myopts.copy() + # For automatic resume, we need to prevent + # any of bad_resume_opts from leaking in + # via EMERGE_DEFAULT_OPTS. + resume_opts["--ignore-default-opts"] = True + for myopt, myarg in resume_opts.iteritems(): if myopt not in bad_resume_opts: if myarg is True: fetch_args.append(myopt) @@ -2936,6 +2989,8 @@ class MergeTask(object): portage.process.spawn(fetch_args, env=fetch_env, fd_pipes=fd_pipes, returnpid=True) logfile.close() # belongs to the spawned process + del fetch_log, logfile, fd_pipes, fetch_env, fetch_args, \ + resume_opts mergecount=0 for x in mymergelist: @@ -3184,7 +3239,12 @@ class MergeTask(object): mtimedb.commit() portage.run_exitfuncs() mynewargv=[sys.argv[0],"--resume"] - for myopt, myarg in self.myopts.iteritems(): + resume_opts = self.myopts.copy() + # For automatic resume, we need to prevent + # any of bad_resume_opts from leaking in + # via EMERGE_DEFAULT_OPTS. + resume_opts["--ignore-default-opts"] = True + for myopt, myarg in resume_opts.iteritems(): if myopt not in bad_resume_opts: if myarg is True: mynewargv.append(myopt) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index e4aa7df18..7b46f7cfd 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2365,10 +2365,19 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", writemsg_stdout("\n", noiselevel=-1) else: if use_locks and can_fetch: + waiting_msg = None + if "parallel-fetch" in features: + waiting_msg = ("Downloading '%s'... " + \ + "see "+EPREFIX+"/var/log/emerge-fetch.log for details.") % myfile if locks_in_subdir: - file_lock = portage.locks.lockfile(mysettings["DISTDIR"]+"/"+locks_in_subdir+"/"+myfile,wantnewlockfile=1) + file_lock = portage.locks.lockfile( + os.path.join(mysettings["DISTDIR"], + locks_in_subdir, myfile), wantnewlockfile=1, + waiting_msg=waiting_msg) else: - file_lock = portage.locks.lockfile(mysettings["DISTDIR"]+"/"+myfile,wantnewlockfile=1) + file_lock = portage.locks.lockfile( + myfile_path, wantnewlockfile=1, + waiting_msg=waiting_msg) try: if not listonly: if fsmirrors and not os.path.exists(myfile_path): @@ -4366,7 +4375,7 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None): mykey=mycpv if mydb and virts and mykey in virts: writemsg("mydb.__class__: %s\n" % (mydb.__class__), 1) - if type(mydb)==types.InstanceType: + if hasattr(mydb, "cp_list"): if not mydb.cp_list(mykey, use_cache=use_cache): writemsg("virts[%s]: %s\n" % (str(mykey),virts[mykey]), 1) mykey_orig = mykey[:] diff --git a/pym/portage/const.py b/pym/portage/const.py index f5b42d073..9156eca04 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -31,7 +31,7 @@ PROFILE_PATH = EPREFIX+"/etc/make.profile" LOCALE_DATA_PATH = PORTAGE_BASE_PATH+"/locale" EBUILD_SH_BINARY = PORTAGE_BIN_PATH+"/ebuild.sh" -MISC_SH_BINARY = PORTAGE_BIN_PATH + "/misc-functions.sh" +MISC_SH_BINARY = PORTAGE_BIN_PATH+"/misc-functions.sh" SANDBOX_BINARY = EPREFIX+"/usr/bin/sandbox" BASH_BINARY = "bash" MOVE_BINARY = "mv" diff --git a/pym/portage/data.py b/pym/portage/data.py index 39e41f817..de1683295 100644 --- a/pym/portage/data.py +++ b/pym/portage/data.py @@ -6,8 +6,8 @@ if not hasattr(__builtins__, "set"): from sets import Set as set -import os,pwd,grp -import portage.const +import os, sys, pwd, grp +from portage.const import wheelgid, rootuid, portageuser, portagegroup from portage.util import writemsg from portage.output import green,red from portage.output import create_color_func @@ -78,10 +78,9 @@ def portage_group_warning(): secpass=0 uid=os.getuid() -wheelgid=portage.const.wheelgid -wheelgroup=grp.getgrgid(portage.const.wheelgid)[0] +wheelgroup=grp.getgrgid(wheelgid)[0] -if uid==0 or uid==int(portage.const.rootuid): +if uid==0 or uid==int(rootuid): secpass=2 try: if (not secpass) and (wheelgid in os.getgroups()): @@ -95,15 +94,15 @@ except KeyError: #Discover the uid and gid of the portage user/group try: - portage_uid=pwd.getpwnam(portage.const.portageuser)[2] - portage_gid=grp.getgrnam(portage.const.portagegroup)[2] + portage_uid=pwd.getpwnam(portageuser)[2] + portage_gid=grp.getgrnam(portagegroup)[2] if secpass < 1 and portage_gid in os.getgroups(): secpass=1 except KeyError: portage_uid=0 portage_gid=wheelgid writemsg("\n") - writemsg( red("portage: "+portage.const.portageuser+" user or group missing. Please update baselayout\n")) + writemsg( red("portage: "+portageuser+" user or group missing. Please update baselayout\n")) writemsg( red(" and merge portage user(250) and group(250) into your passwd\n")) writemsg( red(" and group files. Non-root compilation is disabled until then.\n")) writemsg( " Also note that non-root/wheel users will need to be added to\n") @@ -120,7 +119,7 @@ if secpass >= 2: # Get a list of group IDs for the portage user. Do not use grp.getgrall() # since it is known to trigger spurious SIGPIPE problems with nss_ldap. from commands import getstatusoutput - mystatus, myoutput = getstatusoutput("id -G " + portage.const.portageuser) + mystatus, myoutput = getstatusoutput("id -G " + portageuser) if mystatus == os.EX_OK: for x in myoutput.split(): try: diff --git a/pym/portage/locks.py b/pym/portage/locks.py index da82e65cc..aad682574 100644 --- a/pym/portage/locks.py +++ b/pym/portage/locks.py @@ -18,7 +18,7 @@ def lockdir(mydir): def unlockdir(mylock): return unlockfile(mylock) -def lockfile(mypath,wantnewlockfile=0,unlinkfile=0): +def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None): """Creates all dirs upto, the given dir. Creates a lockfile for the given directory as the file: directoryname+'.portage.lockfile'.""" import fcntl @@ -77,10 +77,13 @@ def lockfile(mypath,wantnewlockfile=0,unlinkfile=0): raise if e.errno == errno.EAGAIN: # resource temp unavailable; eg, someone beat us to the lock. - if type(mypath) == types.IntType: - print "waiting for lock on fd %i" % myfd - else: - print "waiting for lock on %s" % lockfilename + if waiting_msg is None: + if isinstance(mypath, int): + print "waiting for lock on fd %i" % myfd + else: + print "waiting for lock on %s" % lockfilename + elif waiting_msg: + print waiting_msg # try for the exclusive lock now. fcntl.lockf(myfd,fcntl.LOCK_EX) elif e.errno == errno.ENOLCK: diff --git a/pym/portage/versions.py b/pym/portage/versions.py index 834dff2b6..561bc5cae 100644 --- a/pym/portage/versions.py +++ b/pym/portage/versions.py @@ -263,8 +263,6 @@ def pkgsplit(mypkg,silent=1): pkgcache[mypkg]=None return None -_valid_category = re.compile("^\w[\w-]*") - catcache={} def catpkgsplit(mydata,silent=1): """ @@ -284,7 +282,6 @@ def catpkgsplit(mydata,silent=1): """ # Categories may contain a-zA-z0-9+_- but cannot start with - - global _valid_category import portage.dep try: if not catcache[mydata]: @@ -298,9 +295,6 @@ def catpkgsplit(mydata,silent=1): retval=["null"] p_split=pkgsplit(mydata,silent=silent) elif len(mysplit)==2: - if portage.dep._dep_check_strict and \ - not _valid_category.match(mysplit[0]): - raise InvalidData("Invalid category in %s" %mydata ) retval=[mysplit[0]] p_split=pkgsplit(mysplit[1],silent=silent) if not p_split: -- 2.26.2