From bfad1f862d99de33262be8c48f9caf453a503a41 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 25 Aug 2007 10:50:06 +0000 Subject: [PATCH] Merged from trunk 7646:7695 | 7647 | Fix ACCEPT_KEYWORDS validation so that ** is valid. | | zmedico | | | 7651 | Just use os.listdir() since portage.listdir() is | | zmedico | unnecessary. | | 7652 | Try harder to figure out what OS we are running | | WarnerBro | | | 7653 | Bug #189743 - Add .jpeg to the list of extensions | | zmedico | supported by dohtml. | | 7655 | Show cache miss/hit counts all on one line, using \r to | | zmedico | return to the beginning of he line and rewrite it each | | | time that a counter is updated. Thanks to Mike "Fuzzy" | | | Partin for this patch (submitted on | | | the gentoo-portage-dev mailing list). | | 7656 | Send cache hit/miss status updates to stdout instead of | | zmedico | stderr and add some more flush() calls. | | 7659 | Make --info detect more /etc/*-release files as suggested | | zmedico | by grobian. | | 7661 | Bug #188703 - Don't adjust permissions in ${T} unless | | zmedico | userpriv is enabled. | | 7663 | Pass complete package metadata from depgraph.select_dep() | | zmedico | into create() in order to minimize aux_get calls. | | 7664 | Make best() return early when it's only given one | | zmedico | packages. | | 7665 | Fix update_dbentry() to process version components | | zmedico | properly before passing them into ververify(). | | 7666 | Make depgraph.select_dep() reuse cached metadata from | | zmedico | nodes that have already been added to the graph. | | 7671 | Skip --newuse checks for nodes that have already been | | zmedico | added to the graph as "merge" nodes. | | 7672 | Prevent output from being flushed to the console too | | zmedico | frequently in dir_get_metadata(). | | 7674 | Use INFORM to colorize the package name. | | zmedico | | | 7676 | Fix a typo from r7663. | | zmedico | | | 7678 | Bug #189791 - Send errors to /dev/null when debug-print() | | zmedico | fails to write to ${T}/eclass-debug.log. | | 7680 | Make depgraph.create() avoid metadata lookups for binary | | zmedico | packages and ebuilds in cases where an installed package | | | has already been added to the graph. This greatly improves | | | performance when the graph contains a large number of | | | installed packages. | | 7682 | In config.regenerate(), skip loading /etc/profile.env if | | zmedico | it's mtime hasn't changed. | | 7684 | Bug #189791 - Define EBUILD_PHASE=unpack during the | | zmedico | nofetch phase since otherwise we get EBUILD_PHASE=merge | | | which isn't in our list of valid EBUILD_PHASES recognized | | | by elog. | | 7686 | Make depgraph.select_dep() node reuse work when in | | zmedico | --usepkgonly mode. | | 7688 | Add * and ~* to the list of valid values for | | zmedico | ACCEPT_KEYWORDS. | | 7690 | Fix a typo. | | zmedico | | | 7691 | Fix another typo. | | zmedico | | | 7692 | Use a config.reload() method to explicitly reload | | zmedico | /etc/profile.env when setting up the ebuild environment. | | 7693 | Call config.regenerate() after config.reload() so that the | | zmedico | incrementals are restacked. | | 7694 | Don't cache results from match_from_list() since | | zmedico | *dbapi.match() caches do the job. | | 7695 | Remove pointless information from --info output | | genone | | svn path=/main/branches/prefix/; revision=7698 --- bin/dohtml | 7 +- bin/ebuild.sh | 2 +- man/ebuild.5 | 2 +- pym/emerge/__init__.py | 143 ++++++++++++++---------- pym/portage/__init__.py | 25 +++-- pym/portage/dep.py | 11 +- pym/portage/elog/messages.py | 10 +- pym/portage/elog/mod_echo.py | 5 +- pym/portage/getbinpkg.py | 55 +++++++-- pym/portage/tests/news/test_NewsItem.py | 4 +- pym/portage/update.py | 9 +- pym/portage/versions.py | 6 +- 12 files changed, 173 insertions(+), 106 deletions(-) diff --git a/bin/dohtml b/bin/dohtml index 517683f71..b0fce399a 100755 --- a/bin/dohtml +++ b/bin/dohtml @@ -14,8 +14,8 @@ # Detailed usage: # dohtml # - will install the files in the list of files (space-separated list) into -# /usr/share/doc/${PF}/html, provided the file ends in .html, .png, .jpg -# or .css +# /usr/share/doc/${PF}/html, provided the file ends in .htm, .html, .css, +# .js, ,gif, .jpeg, .jpg, or .png. # dohtml -r # - will do as 'dohtml', but recurse into all directories, as long as the # directory name is not CVS @@ -80,7 +80,8 @@ class OptionsClass: if os.environ.has_key("_E_DOCDESTTREE_"): self.DOCDESTTREE = os.environ["_E_DOCDESTTREE_"] - self.allowed_exts = [ 'png', 'gif', 'html', 'htm', 'jpg', 'css', 'js' ] + self.allowed_exts = [ 'htm', 'html', 'css', 'js', + 'gif', 'jpeg', 'jpg', 'png' ] self.allowed_files = [] self.disallowed_dirs = [ 'CVS' ] self.recurse = False diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 4dd3fd997..d46a48caa 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1131,7 +1131,7 @@ debug-print() { fi # default target - echo "$1" >> "${T}/eclass-debug.log" + echo "$1" 2>/dev/null >> "${T}/eclass-debug.log" # let the portage user own/write to this file chmod g+w "${T}/eclass-debug.log" &>/dev/null diff --git a/man/ebuild.5 b/man/ebuild.5 index 868d1d73d..86d1be700 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -775,7 +775,7 @@ Performs the ln command as either a hard link or symlink. .TP \fBdohtml\fR \fI [\-a filetypes] [\-r] [\-x list\-of\-dirs\-to\-ignore] [list\-of\-files\-and\-dirs]\fR Installs the files in the list of files (space\-separated list) into -/usr/share/doc/${PF}/html provided the file ends in .html, .htm, .gif, .jpg, .png, .js, or .css. +/usr/share/doc/${PF}/html provided the file ends in .htm, .html, .css, .js, .gif, .jpeg, .jpg, or .png. Setting \fI\-a\fR limits what types of files will be included, \fI\-A\fR appends to the default list, setting \fI\-x\fR sets which dirs to exclude (CVS excluded by default), \fI\-r\fR sets recursive. diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 5241e8350..075e2eb87 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -1151,7 +1151,7 @@ class depgraph(object): return flags return None - def create(self, mybigkey, myparent=None, addme=1, myuse=None, + def create(self, mybigkey, myparent=None, addme=1, metadata=None, priority=DepPriority(), rev_dep=False, arg=None): """ Fills the digraph with nodes comprised of packages to merge. @@ -1167,7 +1167,6 @@ class depgraph(object): # unused parameters rev_dep = False - myuse = None mytype, myroot, mykey = mybigkey @@ -1191,16 +1190,13 @@ class depgraph(object): # directive, otherwise we add a "merge" directive. mydbapi = self.trees[myroot][self.pkg_tree_map[mytype]].dbapi - metadata = dict(izip(self._mydbapi_keys, - mydbapi.aux_get(mykey, self._mydbapi_keys))) - if mytype == "ebuild": - pkgsettings.setcpv(mykey, mydb=portdb) - metadata["USE"] = pkgsettings["USE"] - myuse = pkgsettings["USE"].split() - else: - # The myuse parameter to this method is deprecated, so get it - # directly from the metadata here. - myuse = metadata["USE"].split() + if metadata is None: + metadata = dict(izip(self._mydbapi_keys, + mydbapi.aux_get(mykey, self._mydbapi_keys))) + if mytype == "ebuild": + pkgsettings.setcpv(mykey, mydb=portdb) + metadata["USE"] = pkgsettings["USE"] + myuse = metadata["USE"].split() if not arg and myroot == self.target_root: try: @@ -1225,6 +1221,10 @@ class depgraph(object): if "empty" not in self.myparams and vardbapi.cpv_exists(mykey): merging=0 + merge_node = (mytype, myroot, mykey, "merge") + if self.digraph.contains(merge_node): + merging = 1 + """ If we aren't merging, perform the --newuse check. If the package has new iuse flags or different use flags then if --newuse is specified, we need to merge the package. """ @@ -1452,8 +1452,7 @@ class depgraph(object): print colorize("BAD", "\n*** You need to adjust PKGDIR to emerge this package.\n") return 0, myfavorites if not self.create(["binary", myroot, mykey], - None, "--onlydeps" not in self.myopts, - myuse=mytbz2.getelements("USE"), arg=x): + addme=("--onlydeps" not in self.myopts), arg=x): return (0,myfavorites) arg_atoms.append((x, "="+mykey)) elif ext==".ebuild": @@ -1727,7 +1726,7 @@ class depgraph(object): for x in mymerge: selected_pkg = None if x[0]=="!": - selected_pkg = ["blocks", myroot, x[1:], None] + selected_pkg = (["blocks", myroot, x[1:]], None) else: #We are not processing a blocker but a normal dependency if myparent: @@ -1744,11 +1743,30 @@ class depgraph(object): matched_packages = [] myeb_matches = portdb.xmatch("match-visible", x) myeb = None - if "--usepkgonly" not in self.myopts: + myeb_pkg = None + metadata = None + existing_node = None + if myeb_matches: myeb = portage.best(myeb_matches) + # For best performance, try to reuse an exising node + # and it's cached metadata. The portdbapi caches SLOT + # metadata in memory so it's really only pulled once. + slot_atom = "%s:%s" % (portage.dep_getkey(myeb), + portdb.aux_get(myeb, ["SLOT"])[0]) + existing_node = self._slot_node_map[myroot].get(slot_atom) + if existing_node: + e_type, myroot, e_cpv, e_status = existing_node + metadata = dict(izip(self._mydbapi_keys, + self.mydbapi[myroot].aux_get(e_cpv, self._mydbapi_keys))) + cpv_slot = "%s:%s" % (e_cpv, metadata["SLOT"]) + if portage.match_from_list(x, [cpv_slot]): + matched_packages.append( + ([e_type, myroot, e_cpv], metadata)) + else: + existing_node = None - myeb_pkg=None - if "--usepkg" in self.myopts: + if not existing_node and \ + "--usepkg" in self.myopts: # The next line assumes the binarytree has been populated. # XXX: Need to work out how we use the binary tree with roots. myeb_pkg_matches = bindb.match(x) @@ -1759,13 +1777,35 @@ class depgraph(object): not portdb.cpv_exists(pkg)] if myeb_pkg_matches: myeb_pkg = portage.best(myeb_pkg_matches) - - if myeb_pkg and \ + # For best performance, try to reuse an exising node + # and it's cached metadata. The bindbapi caches SLOT + # metadata in memory so it's really only pulled once. + slot_atom = "%s:%s" % (portage.dep_getkey(myeb_pkg), + bindb.aux_get(myeb_pkg, ["SLOT"])[0]) + existing_node = self._slot_node_map[myroot].get(slot_atom) + if existing_node: + e_type, myroot, e_cpv, e_status = existing_node + metadata = dict(izip(self._mydbapi_keys, + self.mydbapi[myroot].aux_get(e_cpv, self._mydbapi_keys))) + cpv_slot = "%s:%s" % (e_cpv, metadata["SLOT"]) + if portage.match_from_list(x, [cpv_slot]): + myeb_pkg = None + matched_packages.append( + ([e_type, myroot, e_cpv], metadata)) + else: + existing_node = None + if not existing_node: + # For best performance, avoid pulling + # metadata whenever possible. + metadata = dict(izip(self._mydbapi_keys, + bindb.aux_get(myeb_pkg, self._mydbapi_keys))) + + if not existing_node and \ + myeb_pkg and \ ("--newuse" in self.myopts or \ "--reinstall" in self.myopts): - iuses = set(filter_iuse_defaults( - bindb.aux_get(myeb_pkg, ["IUSE"])[0].split())) - old_use = bindb.aux_get(myeb_pkg, ["USE"])[0].split() + iuses = set(filter_iuse_defaults(metadata["IUSE"].split())) + old_use = metadata["USE"].split() mydb = None if "--usepkgonly" not in self.myopts and myeb: mydb = portdb @@ -1785,15 +1825,18 @@ class depgraph(object): forced_flags, old_use, iuses, now_use, cur_iuse): myeb_pkg = None if myeb_pkg: - binpkguseflags = \ - self.trees[myroot]["bintree"].dbapi.aux_get( - myeb_pkg, ["USE"])[0].split() matched_packages.append( - ["binary", myroot, myeb_pkg, binpkguseflags]) - - if "--usepkgonly" not in self.myopts and myeb_matches: + (["binary", myroot, myeb_pkg], metadata)) + + if not existing_node and \ + myeb and \ + "--usepkgonly" not in self.myopts: + metadata = dict(izip(self._mydbapi_keys, + portdb.aux_get(myeb, self._mydbapi_keys))) + pkgsettings.setcpv(myeb, mydb=portdb) + metadata["USE"] = pkgsettings["USE"] matched_packages.append( - ["ebuild", myroot, myeb, None]) + (["ebuild", myroot, myeb], metadata)) if not matched_packages and \ not (arg and "selective" not in self.myparams): @@ -1810,10 +1853,10 @@ class depgraph(object): if myeb_inst_matches: myeb_inst = portage.best(myeb_inst_matches) if myeb_inst: - binpkguseflags = vardb.aux_get( - myeb_inst, ["USE"])[0].split() + metadata = dict(izip(self._mydbapi_keys, + vardb.aux_get(myeb_inst, self._mydbapi_keys))) matched_packages.append( - ["installed", myroot, myeb_inst, binpkguseflags]) + (["installed", myroot, myeb_inst], metadata)) if not matched_packages: if raise_on_missing: @@ -1887,46 +1930,33 @@ class depgraph(object): return 0 if "--debug" in self.myopts: - for pkg in matched_packages: + for pkg, metadata in matched_packages: print (pkg[0] + ":").rjust(10), pkg[2] if len(matched_packages) > 1: bestmatch = portage.best( - [pkg[2] for pkg in matched_packages]) + [pkg[2] for pkg, metadata in matched_packages]) matched_packages = [pkg for pkg in matched_packages \ - if pkg[2] == bestmatch] + if pkg[0][2] == bestmatch] # ordered by type preference ("ebuild" type is the last resort) selected_pkg = matched_packages[0] - pkgtype, myroot, mycpv, myuse = selected_pkg - mydbapi = self.trees[myroot][self.pkg_tree_map[pkgtype]].dbapi - slot_atom = "%s:%s" % (portage.dep_getkey(mycpv), - mydbapi.aux_get(mycpv, ["SLOT"])[0]) - existing_node = self._slot_node_map[myroot].get( - slot_atom, None) - if existing_node: - e_type, myroot, e_cpv, e_status = existing_node - if portage.match_from_list(x, [e_cpv]): - # The existing node can be reused. - # Just pass in None for myuse since - # self.create() doesn't use it anymore. - selected_pkg = [e_type, myroot, e_cpv, None] if myparent: #we are a dependency, so we want to be unconditionally added mypriority = priority.copy() if vardb.match(x): mypriority.satisfied = True - if not self.create(selected_pkg[0:3], myparent, - myuse=selected_pkg[-1], priority=mypriority, + if not self.create(selected_pkg[0], myparent=myparent, + metadata=selected_pkg[1], priority=mypriority, rev_dep=rev_deps, arg=arg): return 0 else: #if mysource is not set, then we are a command-line dependency and should not be added #if --onlydeps is specified. - if not self.create(selected_pkg[0:3], myparent, + if not self.create(selected_pkg[0], myparent=myparent, addme=("--onlydeps" not in self.myopts), - myuse=selected_pkg[-1], rev_dep=rev_deps, arg=arg): + metadata=selected_pkg[1], rev_dep=rev_deps, arg=arg): return 0 if "--debug" in self.myopts: @@ -4372,6 +4402,7 @@ def post_emerge(trees, mtimedb, retval): # Load the most current variables from ${ROOT}/etc/profile.env settings.unlock() + settings.reload() settings.regenerate() settings.lock() @@ -5183,12 +5214,6 @@ def action_info(settings, trees, myopts, myfiles): print header_title.rjust(int(header_width/2 + len(header_title)/2)) print header_width * "=" print "System uname: "+unameout - gentoo_release = portage.grabfile(os.path.join( - settings["PORTAGE_CONFIGROOT"] + EPREFIX, "etc", "gentoo-release")) - if gentoo_release: - print gentoo_release[0] - else: - print "Unknown Host Operating System" lastSync = portage.grabfile(os.path.join( settings["PORTDIR"], "metadata", "timestamp.chk")) print "Timestamp of tree:", diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 1d1aeeb91..e36bff6a2 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1585,7 +1585,9 @@ class config(object): writemsg("--- 'profiles/arch.list' is empty or not available. Empty portage tree?\n") else: for group in groups: - if group not in archlist and group[0] != '-': + if group not in archlist and \ + not (group.startswith("-") and group[1:] in archlist) and \ + group not in ("*", "~*", "**"): writemsg("!!! INVALID ACCEPT_KEYWORDS: %s\n" % str(group), noiselevel=-1) @@ -1910,6 +1912,14 @@ class config(object): if modified: self.virtuals = self.__getvirtuals_compile() + def reload(self): + """Reload things like /etc/profile.env that can change during runtime.""" + env_d_filename = os.path.join(self["ROOT"], EPREFIX_LSTRIP, "etc", "profile.env") + self.configdict["env.d"].clear() + env_d = getconfig(env_d_filename, expand=False) + if env_d: + # env_d will be None if profile.env doesn't exist. + self.configdict["env.d"].update(env_d) def regenerate(self,useonly=0,use_cache=1): """ @@ -1936,14 +1946,6 @@ class config(object): else: self.already_in_regenerate = 1 - # We grab the latest profile.env here since it changes frequently. - self.configdict["env.d"].clear() - env_d = getconfig( - os.path.join(self["ROOT"], EPREFIX_LSTRIP, "etc", "profile.env"), expand=False) - if env_d: - # env_d will be None if profile.env doesn't exist. - self.configdict["env.d"].update(env_d) - if useonly: myincrementals=["USE"] else: @@ -3001,6 +3003,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", "ebuild's files must be downloaded" print "!!! manually. See the comments in" + \ " the ebuild for more information.\n" + mysettings["EBUILD_PHASE"] = "unpack" spawn(EBUILD_SH_BINARY + " nofetch", mysettings) elif listonly: continue @@ -3247,7 +3250,8 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None): phase_retval = spawn(actionmap[mydo]["cmd"] % mydo, mysettings, debug=debug, logfile=logfile, **kwargs) mysettings["EBUILD_PHASE"] = "" - if not kwargs["droppriv"] and secpass >= 2: + if "userpriv" in mysettings.features and \ + not kwargs["droppriv"] and secpass >= 2: """ Privileged phases may have left files that need to be made writable to a less privileged user.""" apply_recursive_permissions(mysettings["T"], @@ -3311,6 +3315,7 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m detects a package-specific change in config. For the ebuild environment, a reset call is forced in order to ensure that the latest env.d variables are used.""" + mysettings.reload() mysettings.reset(use_cache=use_cache) mysettings.setcpv(mycpv, use_cache=use_cache, mydb=mydbapi) diff --git a/pym/portage/dep.py b/pym/portage/dep.py index fec15332f..da0bb6427 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -629,8 +629,6 @@ def best_match_to_list(mypkg, mylist): bestm = x return bestm -_match_from_list_cache = {} - def match_from_list(mydep, candidate_list): """ Searches list for entries that matches the package. @@ -643,12 +641,6 @@ def match_from_list(mydep, candidate_list): @return: A list of package atoms that match the given package atom """ - global _match_from_list_cache - cache_key = (mydep, tuple(candidate_list)) - mylist = _match_from_list_cache.get(cache_key, None) - if mylist is not None: - return mylist[:] - from portage.util import writemsg if mydep[0] == "!": mydep = mydep[1:] @@ -752,5 +744,4 @@ def match_from_list(mydep, candidate_list): continue mylist.append(x) - _match_from_list_cache[cache_key] = mylist - return mylist[:] + return mylist diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py index 84391adc1..7c860729b 100644 --- a/pym/portage/elog/messages.py +++ b/pym/portage/elog/messages.py @@ -7,17 +7,19 @@ from portage.output import colorize from portage.const import EBUILD_PHASES from portage.util import writemsg -from portage import listdir - import os def collect_ebuild_messages(path): """ Collect elog messages generated by the bash logging function stored at 'path'. """ - mylogfiles = listdir(path) + mylogfiles = None + try: + mylogfiles = os.listdir(path) + except OSError: + pass # shortcut for packages without any messages - if len(mylogfiles) == 0: + if not mylogfiles: return {} # exploit listdir() file order so we process log entries in chronological order mylogfiles.reverse() diff --git a/pym/portage/elog/mod_echo.py b/pym/portage/elog/mod_echo.py index e9bd4bd38..6a12c0364 100644 --- a/pym/portage/elog/mod_echo.py +++ b/pym/portage/elog/mod_echo.py @@ -3,7 +3,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ -from portage.output import EOutput +from portage.output import EOutput, colorize from portage.const import EBUILD_PHASES _items = [] @@ -22,7 +22,8 @@ def finalize(mysettings=None): if mysettings["ROOT"] != "/": root_msg = " merged to %s" % mysettings["ROOT"] print - printer.einfo("Messages for package %s%s:" % (key, root_msg)) + printer.einfo("Messages for package %s%s:" % \ + (colorize("INFORM", key), root_msg)) print for phase in EBUILD_PHASES: if phase not in logentries: diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index e7bfaa039..93d43db09 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -459,11 +459,12 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= makepickle = CACHE_PATH+"/metadata.idx.most_recent" conn,protocol,address,params,headers = create_conn(baseurl, conn) - + out = sys.stdout try: metadatafile = open(CACHE_PATH+"/remote_metadata.pickle") metadata = cPickle.load(metadatafile) - sys.stderr.write("Loaded metadata pickle.\n") + out.write("Loaded metadata pickle.\n") + out.flush() metadatafile.close() except (cPickle.UnpicklingError, OSError, IOError, EOFError): metadata = {} @@ -503,10 +504,12 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= sys.stderr.write("--- "+str(e)+"\n") if trynum < 3: sys.stderr.write("Retrying...\n") + sys.stderr.flush() mytempfile.close() continue if match_in_array([mfile],suffix=".gz"): - sys.stderr.write("gzip'd\n") + out.write("gzip'd\n") + out.flush() try: import gzip mytempfile.seek(0) @@ -517,6 +520,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= except Exception, e: mytempfile.close() sys.stderr.write("!!! Failed to use gzip: "+str(e)+"\n") + sys.stderr.flush() mytempfile.close() try: metadata[baseurl]["data"] = cPickle.loads(data) @@ -524,13 +528,15 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= metadata[baseurl]["indexname"] = mfile metadata[baseurl]["timestamp"] = int(time.time()) metadata[baseurl]["modified"] = 0 # It's not, right after download. - sys.stderr.write("Pickle loaded.\n") + out.write("Pickle loaded.\n") + out.flush() break except SystemExit, e: raise except Exception, e: sys.stderr.write("!!! Failed to read data from index: "+str(mfile)+"\n") sys.stderr.write("!!! "+str(e)+"\n") + sys.stderr.flush() try: metadatafile = open(CACHE_PATH+"/remote_metadata.pickle", "w+") cPickle.dump(metadata,metadatafile) @@ -540,15 +546,40 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= except Exception, e: sys.stderr.write("!!! Failed to write binary metadata to disk!\n") sys.stderr.write("!!! "+str(e)+"\n") + sys.stderr.flush() break # We may have metadata... now we run through the tbz2 list and check. - sys.stderr.write(yellow("cache miss: 'x'")+" --- "+green("cache hit: 'o'")+"\n") + + class CacheStats(object): + from time import time + def __init__(self, out): + self.misses = 0 + self.hits = 0 + self.last_update = 0 + self.out = out + self.min_display_latency = 0.2 + def update(self): + cur_time = self.time() + if cur_time - self.last_update >= self.min_display_latency: + self.last_update = cur_time + self.display() + def display(self): + self.out.write("\r"+yellow("cache miss: '"+str(self.misses)+"'")+\ + " --- "+green("cache hit: '"+str(self.hits)+"'")) + self.out.flush() + + cache_stats = CacheStats(out) + have_tty = out.isatty() + if have_tty: + cache_stats.display() binpkg_filenames = set() for x in tbz2list: x = os.path.basename(x) binpkg_filenames.add(x) if x not in metadata[baseurl]["data"]: - sys.stderr.write(yellow("x")) + cache_stats.misses += 1 + if have_tty: + cache_stats.update() metadata[baseurl]["modified"] = 1 myid = None for retry in xrange(3): @@ -569,8 +600,12 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= metadata[baseurl]["data"][x] = make_metadata_dict(myid) elif verbose: sys.stderr.write(red("!!! Failed to retrieve metadata on: ")+str(x)+"\n") + sys.stderr.flush() else: - sys.stderr.write(green("o")) + cache_stats.hits += 1 + if have_tty: + cache_stats.update() + cache_stats.display() # Cleanse stale cache for files that don't exist on the server anymore. stale_cache = set(metadata[baseurl]["data"]).difference(binpkg_filenames) if stale_cache: @@ -579,8 +614,9 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= metadata[baseurl]["modified"] = 1 del stale_cache del binpkg_filenames - sys.stderr.write("\n") - + out.write("\n") + out.flush() + try: if metadata[baseurl].has_key("modified") and metadata[baseurl]["modified"]: metadata[baseurl]["timestamp"] = int(time.time()) @@ -596,6 +632,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= except Exception, e: sys.stderr.write("!!! Failed to write binary metadata to disk!\n") sys.stderr.write("!!! "+str(e)+"\n") + sys.stderr.flush() if not keepconnection: conn.close() diff --git a/pym/portage/tests/news/test_NewsItem.py b/pym/portage/tests/news/test_NewsItem.py index 6df871728..1fa357fb8 100644 --- a/pym/portage/tests/news/test_NewsItem.py +++ b/pym/portage/tests/news/test_NewsItem.py @@ -73,8 +73,8 @@ The revdep-rebuild tool is provided by app-portage/gentoolkit. def _processItem( self, item ): - path = os.path.join( settings["PORTAGE_TMPDIR"], str(time.time()) - f = open( os.path.join( path ) + path = os.path.join(settings["PORTAGE_TMPDIR"], str(time.time())) + f = open(path) f.write(item) f.close try: diff --git a/pym/portage/update.py b/pym/portage/update.py index c6484a9aa..9a5b78ba3 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -8,7 +8,8 @@ from portage.util import ConfigProtect, grabfile, new_protect_filename, \ normalize_path, write_atomic, writemsg from portage.exception import DirectoryNotFound, PortageException from portage.versions import ververify -from portage.dep import dep_getkey, get_operator, isvalidatom, isjustname +from portage.dep import dep_getkey, get_operator, isvalidatom, isjustname, \ + remove_slot from portage.const import USER_CONFIG_PATH, WORLD_FILE, EPREFIX ignored_dbentries = ("CONTENTS", "environment.bz2") @@ -20,7 +21,11 @@ def update_dbentry(update_cmd, mycontent): old_value = re.escape(old_value); mycontent = re.sub(old_value+"(:|$|\\s)", new_value+"\\1", mycontent) def myreplace(matchobj): - if ververify(matchobj.group(2)): + # Strip slot and * operator if necessary + # so that ververify works. + ver = remove_slot(matchobj.group(2)) + ver = ver.rstrip("*") + if ververify(ver): return "%s-%s" % (new_value, matchobj.group(2)) else: return "".join(matchobj.groups()) diff --git a/pym/portage/versions.py b/pym/portage/versions.py index 49804311f..131aa628c 100644 --- a/pym/portage/versions.py +++ b/pym/portage/versions.py @@ -328,10 +328,10 @@ def catsplit(mydep): def best(mymatches): """Accepts None arguments; assumes matches are valid.""" - if mymatches is None: - return "" - if not len(mymatches): + if not mymatches: return "" + if len(mymatches) == 1: + return mymatches[0] bestmatch = mymatches[0] p2 = catpkgsplit(bestmatch)[1:] for x in mymatches[1:]: -- 2.26.2