Merged from trunk 8328:8338
authorFabian Groffen <grobian@gentoo.org>
Mon, 29 Oct 2007 21:03:37 +0000 (21:03 -0000)
committerFabian Groffen <grobian@gentoo.org>
Mon, 29 Oct 2007 21:03:37 +0000 (21:03 -0000)
   | 8329    | Sort all dbapi.cp_list() results in ascending order and      |
   | zmedico | cache the results when possible. The order is preserved in   |
   |         | dbapi.match() so those results are also sorted.              |

   | 8331    | Fix cp_list() to prevent it from returning a reference to    |
   | zmedico | one of it's internal lists.                                  |

   | 8332    | Optimize stdout_spinner.update() methods so that they don't  |
   | zmedico | waste cpu time by flushing ouput to the tty too frequently.  |

   | 8333    | Fix SetConfig.getSetAtoms() to stop initializing ignorelist  |
   | zmedico | in the parameter list since otherwise the ignorelist         |
   |         | persists between calls and causes unpredictable results for  |
   |         | callers.                                                     |

   | 8334    | Fix --depclean to use SetConfig.getSetAtoms() and also fix a |
   | zmedico | broken reference to packagesets which is now settings.sets.  |

   | 8335    | Fix PackageSet.getAtoms() and getNonAtoms() so that they     |
   | zmedico | don't return references to private data structures.          |

   | 8336    | Make portdbapi.cp_list() use os.listdir() instead of         |
   | zmedico | portage.listdir() since portage.listdir() has unnecessary    |
   |         | overhead and it's cache isn't needed since cp_list() has     |
   |         | it's own cache.                                              |

   | 8337    | Rewrite the dblink.getcontents() code to use str.split(" ")  |
   | zmedico | for splitting CONTENTS lines so that even file paths that    |
   |         | end with spaces can be handled. This patch makes the fix for |
   |         | bug #196836#c6 more complete. Some code for parsing old      |
   |         | malformed symlink entries has been removed sinces it's       |
   |         | probably not useful or worth maintaining anymore.            |

   | 8338    | Use rstrip("\n") to remove trailing newlines from CONTENTS   |
   | zmedico | lines.                                                       |

svn path=/main/branches/prefix/; revision=8340

pym/_emerge/__init__.py
pym/portage/dbapi/__init__.py
pym/portage/dbapi/porttree.py
pym/portage/dbapi/vartree.py
pym/portage/dbapi/virtual.py
pym/portage/sets/__init__.py
pym/portage/sets/base.py

index 4027ca1cb6e8d6118b6db33424183beb40f1953f..4ba9467d98bf1d60218c8c49e64d8c082f1a08f8 100644 (file)
@@ -53,10 +53,8 @@ from portage.const import EPREFIX, BPREFIX
 from portage.data import secpass
 from portage.util import normalize_path as normpath
 from portage.util import writemsg
-from portage.sets import SetConfig, make_default_config, SETPREFIX
-from portage.sets.profiles import PackagesSystemSet as SystemSet
+from portage.sets import make_default_config, SETPREFIX
 from portage.sets.base import InternalPackageSet
-from portage.sets.files import WorldSet
 
 from itertools import chain, izip
 from UserDict import DictMixin
@@ -94,9 +92,25 @@ class stdout_spinner(object):
                self.update = self.update_twirl
                self.scroll_sequence = self.scroll_msgs[
                        int(time.time() * 100) % len(self.scroll_msgs)]
+               self.last_update = 0
+               self.min_display_latency = 0.05
+
+       def _return_early(self):
+               """
+               Flushing ouput to the tty too frequently wastes cpu time. Therefore,
+               each update* method should return without doing any output when this
+               method returns True.
+               """
+               cur_time = time.time()
+               if cur_time - self.last_update < self.min_display_latency:
+                       return True
+               self.last_update = cur_time
+               return False
 
        def update_basic(self):
                self.spinpos = (self.spinpos + 1) % 500
+               if self._return_early():
+                       return
                if (self.spinpos % 100) == 0:
                        if self.spinpos == 0:
                                sys.stdout.write(". ")
@@ -105,6 +119,8 @@ class stdout_spinner(object):
                sys.stdout.flush()
 
        def update_scroll(self):
+               if self._return_early():
+                       return
                if(self.spinpos >= len(self.scroll_sequence)):
                        sys.stdout.write(darkgreen(" \b\b\b" + self.scroll_sequence[
                                len(self.scroll_sequence) - 1 - (self.spinpos % len(self.scroll_sequence))]))
@@ -115,6 +131,8 @@ class stdout_spinner(object):
 
        def update_twirl(self):
                self.spinpos = (self.spinpos + 1) % len(self.twirl_sequence)
+               if self._return_early():
+                       return
                sys.stdout.write("\b\b " + self.twirl_sequence[self.spinpos])
                sys.stdout.flush()
 
@@ -828,23 +846,6 @@ def perform_global_updates(mycpv, mydb, mycommands):
        if updates:
                mydb.aux_update(mycpv, updates)
 
-def cpv_sort_descending(cpv_list):
-       """Sort in place, returns None."""
-       if len(cpv_list) <= 1:
-               return
-       first_split = portage.catpkgsplit(cpv_list[0])
-       cat = first_split[0]
-       cpv_list[0] = first_split[1:]
-       for i in xrange(1, len(cpv_list)):
-               cpv_list[i] = portage.catpkgsplit(cpv_list[i])[1:]
-       cpv_list.sort(portage.pkgcmp, reverse=True)
-       for i, (pn, ver, rev) in enumerate(cpv_list):
-               if rev == "r0":
-                       cpv = cat + "/" + pn + "-" + ver
-               else:
-                       cpv = cat + "/" + pn + "-" + ver + "-" + rev
-               cpv_list[i] = cpv
-
 def visible(pkgsettings, cpv, metadata, built=False, installed=False):
        """
        Check if a package is visible. This can raise an InvalidDependString
@@ -1359,9 +1360,6 @@ class depgraph(object):
                                        return 0
                                del e
 
-               if "--nodeps" not in self.myopts:
-                       self.spinner.update()
-
                merging = mytype != "installed"
                jbigkey = pkg.digraph_node
 
@@ -1475,6 +1473,8 @@ class depgraph(object):
                elif "recurse" not in self.myparams:
                        return 1
 
+               self.spinner.update()
+
                """ Check DEPEND/RDEPEND/PDEPEND/SLOT
                Pull from bintree if it's binary package, porttree if it's ebuild.
                Binpkg's can be either remote or local. """
@@ -1880,7 +1880,8 @@ class depgraph(object):
                                                # we have to try all of them to prevent the old-style
                                                # virtuals from overriding available new-styles.
                                        continue
-                               cpv_sort_descending(cpv_list)
+                               # descending order
+                               cpv_list.reverse()
                                for cpv in cpv_list:
                                        if filtered_db.cpv_exists(cpv):
                                                continue
@@ -1991,7 +1992,8 @@ class depgraph(object):
                                cpv_list = db.xmatch("match-all", atom)
                        else:
                                cpv_list = db.match(atom)
-                       cpv_sort_descending(cpv_list)
+                       # descending order
+                       cpv_list.reverse()
                        for cpv in cpv_list:
                                try:
                                        metadata = dict(izip(db_keys,
@@ -2110,7 +2112,8 @@ class depgraph(object):
                                        cpv_list = db.xmatch("match-all", atom)
                                else:
                                        cpv_list = db.match(atom)
-                               cpv_sort_descending(cpv_list)
+                               # descending order
+                               cpv_list.reverse()
                                for cpv in cpv_list:
                                        reinstall_for_flags = None
                                        try:
@@ -5729,10 +5732,8 @@ def action_depclean(settings, trees, ldpath_mtimes,
        vardb = dep_check_trees[myroot]["vartree"].dbapi
        # Constrain dependency selection to the installed packages.
        dep_check_trees[myroot]["porttree"] = dep_check_trees[myroot]["vartree"]
-       system_set = SystemSet(settings.profiles)
-       syslist = list(system_set)
-       world_set = WorldSet(myroot)
-       worldlist = list(world_set)
+       syslist = settings.setconfig.getSetAtoms("system")
+       worldlist = settings.setconfig.getSetAtoms("world")
        args_set = InternalPackageSet()
        fakedb = portage.fakedbapi(settings=settings)
        myvarlist = vardb.cpv_all()
@@ -6599,7 +6600,8 @@ def emerge_main():
                                                colorize("INFORM", s)
                                        return 1
                                # TODO: check if the current setname also resolves to a package name
-                               if myaction in ["unmerge", "prune", "clean", "depclean"] and not packagesets[s].supportsOperation("unmerge"):
+                               if myaction in ["unmerge", "prune", "clean", "depclean"] and \
+                                       not settings.sets[s].supportsOperation("unmerge"):
                                        print "emerge: the given set %s does not support unmerge operations" % s
                                        return 1
                                if not settings.setconfig.getSetAtoms(s):
index 329271ee68663e6d1fd3f68193830e997027cc00..0bec4cce5a49162b4e5053440095b1494b8c9f27 100644 (file)
@@ -9,7 +9,7 @@ from portage.locks import unlockfile
 from portage.output import red
 from portage.util import writemsg
 from portage import dep_expand
-from portage.versions import catsplit
+from portage.versions import catpkgsplit, catsplit, pkgcmp
 
 
 class dbapi(object):
@@ -22,6 +22,25 @@ class dbapi(object):
        def cp_list(self, cp, use_cache=1):
                return
 
+       def _cpv_sort_ascending(self, cpv_list):
+               """
+               Use this to sort self.cp_list() results in ascending
+               order. It sorts in place and returns None.
+               """
+               if len(cpv_list) > 1:
+                       first_split = catpkgsplit(cpv_list[0])
+                       cat = first_split[0]
+                       cpv_list[0] = first_split[1:]
+                       for i in xrange(1, len(cpv_list)):
+                               cpv_list[i] = catpkgsplit(cpv_list[i])[1:]
+                       cpv_list.sort(pkgcmp)
+                       for i, (pn, ver, rev) in enumerate(cpv_list):
+                               if rev == "r0":
+                                       cpv = cat + "/" + pn + "-" + ver
+                               else:
+                                       cpv = cat + "/" + pn + "-" + ver + "-" + rev
+                               cpv_list[i] = cpv
+
        def cpv_all(self):
                """Return all CPVs in the db
                Args:
index 4f9b1cbbe3a9dd030503db310d8657465a6e89dc..ec38e0850dfc6a0c36462ee83454d13671734a6f 100644 (file)
@@ -511,7 +511,11 @@ class portdbapi(dbapi):
                else:
                        mytrees = self.porttrees
                for oroot in mytrees:
-                       for x in listdir(oroot+"/"+mycp, EmptyOnError=1, ignorecvs=1):
+                       try:
+                               file_list = os.listdir(os.path.join(oroot, mycp))
+                       except OSError:
+                               continue
+                       for x in file_list:
                                if x.endswith(".ebuild"):
                                        pf = x[:-7]
                                        ps = pkgsplit(pf)
@@ -528,17 +532,7 @@ class portdbapi(dbapi):
                        mylist = d.keys()
                # Always sort in ascending order here since it's handy
                # and the result can be easily cached and reused.
-               if len(mylist) > 1:
-                       for i in xrange(len(mylist)):
-                               mylist[i] = catpkgsplit(mylist[i])[1:]
-                       mylist.sort(pkgcmp)
-                       cat = mysplit[0]
-                       for i, (pn, ver, rev) in enumerate(mylist):
-                               if rev == "r0":
-                                       cpv = cat + "/" + pn + "-" + ver
-                               else:
-                                       cpv = cat + "/" + pn + "-" + ver + "-" + rev
-                               mylist[i] = cpv
+               self._cpv_sort_ascending(mylist)
                if self.frozen and mytree is None:
                        cachelist = mylist[:]
                        self.xcache["cp-list"][mycp] = cachelist
index 98093b9b8d873d2e83e413771eb05e355dcbdffd..05c56263590d90e831f0fe4ea61cb472dfee065b 100644 (file)
@@ -343,7 +343,7 @@ class vardbapi(dbapi):
                if use_cache and self.cpcache.has_key(mycp):
                        cpc = self.cpcache[mycp]
                        if cpc[0] == mystat:
-                               return cpc[1]
+                               return cpc[1][:]
                cat_dir = self.getpath(mysplit[0])
                try:
                        dir_list = os.listdir(cat_dir)
@@ -369,7 +369,8 @@ class vardbapi(dbapi):
                                if ps[0] == mysplit[1]:
                                        returnme.append(mysplit[0]+"/"+x)
                if use_cache:
-                       self.cpcache[mycp] = [mystat,returnme]
+                       self._cpv_sort_ascending(returnme)
+                       self.cpcache[mycp] = [mystat, returnme[:]]
                elif self.cpcache.has_key(mycp):
                        del self.cpcache[mycp]
                return returnme
@@ -969,23 +970,37 @@ class dblink(object):
                                        "file, line %d: '%s'\n" % (pos, contents_file),
                                        noiselevel=-1)
                                continue
-                       mydat = line.split()
-                       correct_split_count = None
-                       if mydat:
-                               correct_split_count = contents_split_counts.get(mydat[0])
-                       if correct_split_count and len(mydat) != correct_split_count:
-                               if mydat[0] == "obj" and \
-                                       len(mydat) > contents_split_counts["obj"]:
-                                       # File name contains spaces. Use field widths to infer the
-                                       # start and end points so that even multiple consecutive
-                                       # spaces are parsed correctly.
-                                       newsplit = ["obj"]
-                                       filename_start = len(mydat[0]) + 1
-                                       filename_end = len(line.rstrip()) - \
-                                               len(mydat[-1]) - len(mydat[-2]) - 2
-                                       newsplit.append(line[filename_start:filename_end])
-                                       newsplit.extend(mydat[-2:])
-                                       mydat = newsplit
+                       line = line.rstrip("\n")
+                       # Split on " " so that even file paths that
+                       # end with spaces can be handled.
+                       mydat = line.split(" ")
+                       entry_type = mydat[0] # empty string if line is empty
+                       correct_split_count = contents_split_counts.get(entry_type)
+                       if correct_split_count and len(mydat) > correct_split_count:
+                               # Apparently file paths contain spaces, so reassemble
+                               # the split have the correct_split_count.
+                               newsplit = [entry_type]
+                               spaces_total = len(mydat) - correct_split_count
+                               if entry_type == "sym":
+                                       try:
+                                               splitter = mydat.index("->", 2, len(mydat) - 2)
+                                       except ValueError:
+                                               writemsg("!!! Unrecognized CONTENTS entry on " + \
+                                                       "line %d: '%s'\n" % (pos, line), noiselevel=-1)
+                                               continue
+                                       spaces_in_path = splitter - 2
+                                       spaces_in_target = spaces_total - spaces_in_path
+                                       newsplit.append(" ".join(mydat[1:splitter]))
+                                       newsplit.append("->")
+                                       target_end = splitter + spaces_in_target + 2
+                                       newsplit.append(" ".join(mydat[splitter + 1:target_end]))
+                                       newsplit.extend(mydat[target_end:])
+                               else:
+                                       path_end = spaces_total + 2
+                                       newsplit.append(" ".join(mydat[1:path_end]))
+                                       newsplit.extend(mydat[path_end:])
+                               mydat = newsplit
+
                        # we do this so we can remove from non-root filesystems
                        # (use the ROOT var to allow maintenance on other partitions)
                        try:
@@ -997,32 +1012,19 @@ class dblink(object):
                                        mydat[1] = os.path.join(myroot, mydat[1].lstrip(os.path.sep))
                                if mydat[0] == "obj":
                                        #format: type, mtime, md5sum
-                                       pkgfiles[mydat[1]] = [mydat[0], mydat[-1], mydat[-2]]
+                                       pkgfiles[mydat[1]] = [mydat[0], mydat[3], mydat[2]]
                                elif mydat[0] == "dir":
                                        #format: type
-                                       pkgfiles[" ".join(mydat[1:])] = [mydat[0] ]
+                                       pkgfiles[mydat[1]] = [mydat[0]]
                                elif mydat[0] == "sym":
                                        #format: type, mtime, dest
-                                       x = len(mydat) - 1
-                                       if (x >= 13) and (mydat[-1][-1] == ')'): # Old/Broken symlink entry
-                                               mydat = mydat[:-10] + [mydat[-10:][stat.ST_MTIME][:-1]]
-                                               writemsg("FIXED SYMLINK LINE: %s\n" % mydat, 1)
-                                               x = len(mydat) - 1
-                                       splitter = -1
-                                       while (x >= 0):
-                                               if mydat[x] == "->":
-                                                       splitter = x
-                                                       break
-                                               x = x - 1
-                                       if splitter == -1:
-                                               return None
-                                       pkgfiles[" ".join(mydat[1:splitter])] = [mydat[0], mydat[-1], " ".join(mydat[(splitter+1):-1])]
+                                       pkgfiles[mydat[1]] = [mydat[0], mydat[4], mydat[3]]
                                elif mydat[0] == "dev":
                                        #format: type
-                                       pkgfiles[" ".join(mydat[1:])] = [mydat[0] ]
+                                       pkgfiles[mydat[1]] = [mydat[0]]
                                elif mydat[0]=="fif":
                                        #format: type
-                                       pkgfiles[" ".join(mydat[1:])] = [mydat[0]]
+                                       pkgfiles[mydat[1]] = [mydat[0]]
                                else:
                                        writemsg("!!! Unrecognized CONTENTS entry on " + \
                                                "line %d: '%s'\n" % (pos, line), noiselevel=-1)
index 65ac0a8d03405b4074ea5e2c86f26dd2351cd3f8..fe93562127168c00646899e0c3f1ab2aae29c9ae 100644 (file)
@@ -41,10 +41,17 @@ class fakedbapi(dbapi):
                return self.cpvdict.has_key(mycpv)
 
        def cp_list(self, mycp, use_cache=1):
-               if not self.cpdict.has_key(mycp):
-                       return []
-               else:
-                       return self.cpdict[mycp]
+               cachelist = self._match_cache.get(mycp)
+               # cp_list() doesn't expand old-style virtuals
+               if cachelist and cachelist[0].startswith(mycp):
+                       return cachelist[:]
+               cpv_list = self.cpdict.get(mycp)
+               if cpv_list is None:
+                       cpv_list = []
+               self._cpv_sort_ascending(cpv_list)
+               if not (not cpv_list and mycp.startswith("virtual/")):
+                       self._match_cache[mycp] = cpv_list
+               return cpv_list[:]
 
        def cp_all(self):
                return list(self.cpdict)
index 9f562820b4dcb59870d0854707ee96555d0ee57b..940aa10204da6be927627b1bdf26fd7a6202e0f6 100644 (file)
@@ -94,14 +94,17 @@ class SetConfig(SafeConfigParser):
                        self.aliases = shortnames
                return self.aliases
 
-       def getSetAtoms(self, setname, ignorelist=[]):
+       def getSetAtoms(self, setname, ignorelist=None):
                myset = self.getSetsWithAliases()[setname]
                myatoms = myset.getAtoms()
-               ignorelist.append(setname)
+               if ignorelist is None:
+                       ignorelist = set()
+               ignorelist.add(setname)
                for n in myset.getNonAtoms():
                        if n[0] == SETPREFIX and n[1:] in self.aliases:
                                if n[1:] not in ignorelist:
-                                       myatoms.update(self.getSetAtoms(n[1:]))
+                                       myatoms.update(self.getSetAtoms(n[1:],
+                                               ignorelist=ignorelist))
                return myatoms
 
 def make_default_config(settings, trees):
index c22ee995363d9ae3860b94f27a1c3c4978e6ed41..8e5863cbaa8d4b82efd0bbe5051eaaa7e4b757d2 100644 (file)
@@ -26,7 +26,8 @@ class PackageSet(object):
                self._nonatoms = set()
 
        def __contains__(self, atom):
-               return atom in self.getAtoms() or atom in self._nonatoms
+               self._load()
+               return atom in self._atoms or atom in self._nonatoms
        
        def __iter__(self):
                for x in self.getAtoms():
@@ -36,18 +37,21 @@ class PackageSet(object):
                if not op in OPERATIONS:
                        raise ValueError(op)
                return op in self._operations
-       
-       def getAtoms(self):
+
+       def _load(self):
                if not (self._loaded or self._loading):
                        self._loading = True
                        self.load()
                        self._loaded = True
                        self._loading = False
-               return self._atoms
+
+       def getAtoms(self):
+               self._load()
+               return self._atoms.copy()
 
        def getNonAtoms(self):
-               self.getAtoms()
-               return self._nonatoms
+               self._load()
+               return self._nonatoms.copy()
 
        def _setAtoms(self, atoms):
                atoms = map(str.strip, atoms)