From: Fabian Groffen Date: Mon, 29 Oct 2007 21:03:37 +0000 (-0000) Subject: Merged from trunk 8328:8338 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=9a9b1b6eef8c1d5067e7908732393e3b091cecd2;p=portage.git Merged from trunk 8328:8338 | 8329 | Sort all dbapi.cp_list() results in ascending order and | | zmedico | cache the results when possible. The order is preserved in | | | dbapi.match() so those results are also sorted. | | 8331 | Fix cp_list() to prevent it from returning a reference to | | zmedico | one of it's internal lists. | | 8332 | Optimize stdout_spinner.update() methods so that they don't | | zmedico | waste cpu time by flushing ouput to the tty too frequently. | | 8333 | Fix SetConfig.getSetAtoms() to stop initializing ignorelist | | zmedico | in the parameter list since otherwise the ignorelist | | | persists between calls and causes unpredictable results for | | | callers. | | 8334 | Fix --depclean to use SetConfig.getSetAtoms() and also fix a | | zmedico | broken reference to packagesets which is now settings.sets. | | 8335 | Fix PackageSet.getAtoms() and getNonAtoms() so that they | | zmedico | don't return references to private data structures. | | 8336 | Make portdbapi.cp_list() use os.listdir() instead of | | zmedico | portage.listdir() since portage.listdir() has unnecessary | | | overhead and it's cache isn't needed since cp_list() has | | | it's own cache. | | 8337 | Rewrite the dblink.getcontents() code to use str.split(" ") | | zmedico | for splitting CONTENTS lines so that even file paths that | | | end with spaces can be handled. This patch makes the fix for | | | bug #196836#c6 more complete. Some code for parsing old | | | malformed symlink entries has been removed sinces it's | | | probably not useful or worth maintaining anymore. | | 8338 | Use rstrip("\n") to remove trailing newlines from CONTENTS | | zmedico | lines. | svn path=/main/branches/prefix/; revision=8340 --- diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 4027ca1cb..4ba9467d9 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -53,10 +53,8 @@ from portage.const import EPREFIX, BPREFIX from portage.data import secpass from portage.util import normalize_path as normpath from portage.util import writemsg -from portage.sets import SetConfig, make_default_config, SETPREFIX -from portage.sets.profiles import PackagesSystemSet as SystemSet +from portage.sets import make_default_config, SETPREFIX from portage.sets.base import InternalPackageSet -from portage.sets.files import WorldSet from itertools import chain, izip from UserDict import DictMixin @@ -94,9 +92,25 @@ class stdout_spinner(object): self.update = self.update_twirl self.scroll_sequence = self.scroll_msgs[ int(time.time() * 100) % len(self.scroll_msgs)] + self.last_update = 0 + self.min_display_latency = 0.05 + + def _return_early(self): + """ + Flushing ouput to the tty too frequently wastes cpu time. Therefore, + each update* method should return without doing any output when this + method returns True. + """ + cur_time = time.time() + if cur_time - self.last_update < self.min_display_latency: + return True + self.last_update = cur_time + return False def update_basic(self): self.spinpos = (self.spinpos + 1) % 500 + if self._return_early(): + return if (self.spinpos % 100) == 0: if self.spinpos == 0: sys.stdout.write(". ") @@ -105,6 +119,8 @@ class stdout_spinner(object): sys.stdout.flush() def update_scroll(self): + if self._return_early(): + return if(self.spinpos >= len(self.scroll_sequence)): sys.stdout.write(darkgreen(" \b\b\b" + self.scroll_sequence[ len(self.scroll_sequence) - 1 - (self.spinpos % len(self.scroll_sequence))])) @@ -115,6 +131,8 @@ class stdout_spinner(object): def update_twirl(self): self.spinpos = (self.spinpos + 1) % len(self.twirl_sequence) + if self._return_early(): + return sys.stdout.write("\b\b " + self.twirl_sequence[self.spinpos]) sys.stdout.flush() @@ -828,23 +846,6 @@ def perform_global_updates(mycpv, mydb, mycommands): if updates: mydb.aux_update(mycpv, updates) -def cpv_sort_descending(cpv_list): - """Sort in place, returns None.""" - if len(cpv_list) <= 1: - return - first_split = portage.catpkgsplit(cpv_list[0]) - cat = first_split[0] - cpv_list[0] = first_split[1:] - for i in xrange(1, len(cpv_list)): - cpv_list[i] = portage.catpkgsplit(cpv_list[i])[1:] - cpv_list.sort(portage.pkgcmp, reverse=True) - for i, (pn, ver, rev) in enumerate(cpv_list): - if rev == "r0": - cpv = cat + "/" + pn + "-" + ver - else: - cpv = cat + "/" + pn + "-" + ver + "-" + rev - cpv_list[i] = cpv - def visible(pkgsettings, cpv, metadata, built=False, installed=False): """ Check if a package is visible. This can raise an InvalidDependString @@ -1359,9 +1360,6 @@ class depgraph(object): return 0 del e - if "--nodeps" not in self.myopts: - self.spinner.update() - merging = mytype != "installed" jbigkey = pkg.digraph_node @@ -1475,6 +1473,8 @@ class depgraph(object): elif "recurse" not in self.myparams: return 1 + self.spinner.update() + """ Check DEPEND/RDEPEND/PDEPEND/SLOT Pull from bintree if it's binary package, porttree if it's ebuild. Binpkg's can be either remote or local. """ @@ -1880,7 +1880,8 @@ class depgraph(object): # we have to try all of them to prevent the old-style # virtuals from overriding available new-styles. continue - cpv_sort_descending(cpv_list) + # descending order + cpv_list.reverse() for cpv in cpv_list: if filtered_db.cpv_exists(cpv): continue @@ -1991,7 +1992,8 @@ class depgraph(object): cpv_list = db.xmatch("match-all", atom) else: cpv_list = db.match(atom) - cpv_sort_descending(cpv_list) + # descending order + cpv_list.reverse() for cpv in cpv_list: try: metadata = dict(izip(db_keys, @@ -2110,7 +2112,8 @@ class depgraph(object): cpv_list = db.xmatch("match-all", atom) else: cpv_list = db.match(atom) - cpv_sort_descending(cpv_list) + # descending order + cpv_list.reverse() for cpv in cpv_list: reinstall_for_flags = None try: @@ -5729,10 +5732,8 @@ def action_depclean(settings, trees, ldpath_mtimes, vardb = dep_check_trees[myroot]["vartree"].dbapi # Constrain dependency selection to the installed packages. dep_check_trees[myroot]["porttree"] = dep_check_trees[myroot]["vartree"] - system_set = SystemSet(settings.profiles) - syslist = list(system_set) - world_set = WorldSet(myroot) - worldlist = list(world_set) + syslist = settings.setconfig.getSetAtoms("system") + worldlist = settings.setconfig.getSetAtoms("world") args_set = InternalPackageSet() fakedb = portage.fakedbapi(settings=settings) myvarlist = vardb.cpv_all() @@ -6599,7 +6600,8 @@ def emerge_main(): colorize("INFORM", s) return 1 # TODO: check if the current setname also resolves to a package name - if myaction in ["unmerge", "prune", "clean", "depclean"] and not packagesets[s].supportsOperation("unmerge"): + if myaction in ["unmerge", "prune", "clean", "depclean"] and \ + not settings.sets[s].supportsOperation("unmerge"): print "emerge: the given set %s does not support unmerge operations" % s return 1 if not settings.setconfig.getSetAtoms(s): diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index 329271ee6..0bec4cce5 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -9,7 +9,7 @@ from portage.locks import unlockfile from portage.output import red from portage.util import writemsg from portage import dep_expand -from portage.versions import catsplit +from portage.versions import catpkgsplit, catsplit, pkgcmp class dbapi(object): @@ -22,6 +22,25 @@ class dbapi(object): def cp_list(self, cp, use_cache=1): return + def _cpv_sort_ascending(self, cpv_list): + """ + Use this to sort self.cp_list() results in ascending + order. It sorts in place and returns None. + """ + if len(cpv_list) > 1: + first_split = catpkgsplit(cpv_list[0]) + cat = first_split[0] + cpv_list[0] = first_split[1:] + for i in xrange(1, len(cpv_list)): + cpv_list[i] = catpkgsplit(cpv_list[i])[1:] + cpv_list.sort(pkgcmp) + for i, (pn, ver, rev) in enumerate(cpv_list): + if rev == "r0": + cpv = cat + "/" + pn + "-" + ver + else: + cpv = cat + "/" + pn + "-" + ver + "-" + rev + cpv_list[i] = cpv + def cpv_all(self): """Return all CPVs in the db Args: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 4f9b1cbbe..ec38e0850 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -511,7 +511,11 @@ class portdbapi(dbapi): else: mytrees = self.porttrees for oroot in mytrees: - for x in listdir(oroot+"/"+mycp, EmptyOnError=1, ignorecvs=1): + try: + file_list = os.listdir(os.path.join(oroot, mycp)) + except OSError: + continue + for x in file_list: if x.endswith(".ebuild"): pf = x[:-7] ps = pkgsplit(pf) @@ -528,17 +532,7 @@ class portdbapi(dbapi): mylist = d.keys() # Always sort in ascending order here since it's handy # and the result can be easily cached and reused. - if len(mylist) > 1: - for i in xrange(len(mylist)): - mylist[i] = catpkgsplit(mylist[i])[1:] - mylist.sort(pkgcmp) - cat = mysplit[0] - for i, (pn, ver, rev) in enumerate(mylist): - if rev == "r0": - cpv = cat + "/" + pn + "-" + ver - else: - cpv = cat + "/" + pn + "-" + ver + "-" + rev - mylist[i] = cpv + self._cpv_sort_ascending(mylist) if self.frozen and mytree is None: cachelist = mylist[:] self.xcache["cp-list"][mycp] = cachelist diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 98093b9b8..05c562635 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -343,7 +343,7 @@ class vardbapi(dbapi): if use_cache and self.cpcache.has_key(mycp): cpc = self.cpcache[mycp] if cpc[0] == mystat: - return cpc[1] + return cpc[1][:] cat_dir = self.getpath(mysplit[0]) try: dir_list = os.listdir(cat_dir) @@ -369,7 +369,8 @@ class vardbapi(dbapi): if ps[0] == mysplit[1]: returnme.append(mysplit[0]+"/"+x) if use_cache: - self.cpcache[mycp] = [mystat,returnme] + self._cpv_sort_ascending(returnme) + self.cpcache[mycp] = [mystat, returnme[:]] elif self.cpcache.has_key(mycp): del self.cpcache[mycp] return returnme @@ -969,23 +970,37 @@ class dblink(object): "file, line %d: '%s'\n" % (pos, contents_file), noiselevel=-1) continue - mydat = line.split() - correct_split_count = None - if mydat: - correct_split_count = contents_split_counts.get(mydat[0]) - if correct_split_count and len(mydat) != correct_split_count: - if mydat[0] == "obj" and \ - len(mydat) > contents_split_counts["obj"]: - # File name contains spaces. Use field widths to infer the - # start and end points so that even multiple consecutive - # spaces are parsed correctly. - newsplit = ["obj"] - filename_start = len(mydat[0]) + 1 - filename_end = len(line.rstrip()) - \ - len(mydat[-1]) - len(mydat[-2]) - 2 - newsplit.append(line[filename_start:filename_end]) - newsplit.extend(mydat[-2:]) - mydat = newsplit + line = line.rstrip("\n") + # Split on " " so that even file paths that + # end with spaces can be handled. + mydat = line.split(" ") + entry_type = mydat[0] # empty string if line is empty + correct_split_count = contents_split_counts.get(entry_type) + if correct_split_count and len(mydat) > correct_split_count: + # Apparently file paths contain spaces, so reassemble + # the split have the correct_split_count. + newsplit = [entry_type] + spaces_total = len(mydat) - correct_split_count + if entry_type == "sym": + try: + splitter = mydat.index("->", 2, len(mydat) - 2) + except ValueError: + writemsg("!!! Unrecognized CONTENTS entry on " + \ + "line %d: '%s'\n" % (pos, line), noiselevel=-1) + continue + spaces_in_path = splitter - 2 + spaces_in_target = spaces_total - spaces_in_path + newsplit.append(" ".join(mydat[1:splitter])) + newsplit.append("->") + target_end = splitter + spaces_in_target + 2 + newsplit.append(" ".join(mydat[splitter + 1:target_end])) + newsplit.extend(mydat[target_end:]) + else: + path_end = spaces_total + 2 + newsplit.append(" ".join(mydat[1:path_end])) + newsplit.extend(mydat[path_end:]) + mydat = newsplit + # we do this so we can remove from non-root filesystems # (use the ROOT var to allow maintenance on other partitions) try: @@ -997,32 +1012,19 @@ class dblink(object): mydat[1] = os.path.join(myroot, mydat[1].lstrip(os.path.sep)) if mydat[0] == "obj": #format: type, mtime, md5sum - pkgfiles[mydat[1]] = [mydat[0], mydat[-1], mydat[-2]] + pkgfiles[mydat[1]] = [mydat[0], mydat[3], mydat[2]] elif mydat[0] == "dir": #format: type - pkgfiles[" ".join(mydat[1:])] = [mydat[0] ] + pkgfiles[mydat[1]] = [mydat[0]] elif mydat[0] == "sym": #format: type, mtime, dest - x = len(mydat) - 1 - if (x >= 13) and (mydat[-1][-1] == ')'): # Old/Broken symlink entry - mydat = mydat[:-10] + [mydat[-10:][stat.ST_MTIME][:-1]] - writemsg("FIXED SYMLINK LINE: %s\n" % mydat, 1) - x = len(mydat) - 1 - splitter = -1 - while (x >= 0): - if mydat[x] == "->": - splitter = x - break - x = x - 1 - if splitter == -1: - return None - pkgfiles[" ".join(mydat[1:splitter])] = [mydat[0], mydat[-1], " ".join(mydat[(splitter+1):-1])] + pkgfiles[mydat[1]] = [mydat[0], mydat[4], mydat[3]] elif mydat[0] == "dev": #format: type - pkgfiles[" ".join(mydat[1:])] = [mydat[0] ] + pkgfiles[mydat[1]] = [mydat[0]] elif mydat[0]=="fif": #format: type - pkgfiles[" ".join(mydat[1:])] = [mydat[0]] + pkgfiles[mydat[1]] = [mydat[0]] else: writemsg("!!! Unrecognized CONTENTS entry on " + \ "line %d: '%s'\n" % (pos, line), noiselevel=-1) diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py index 65ac0a8d0..fe9356212 100644 --- a/pym/portage/dbapi/virtual.py +++ b/pym/portage/dbapi/virtual.py @@ -41,10 +41,17 @@ class fakedbapi(dbapi): return self.cpvdict.has_key(mycpv) def cp_list(self, mycp, use_cache=1): - if not self.cpdict.has_key(mycp): - return [] - else: - return self.cpdict[mycp] + cachelist = self._match_cache.get(mycp) + # cp_list() doesn't expand old-style virtuals + if cachelist and cachelist[0].startswith(mycp): + return cachelist[:] + cpv_list = self.cpdict.get(mycp) + if cpv_list is None: + cpv_list = [] + self._cpv_sort_ascending(cpv_list) + if not (not cpv_list and mycp.startswith("virtual/")): + self._match_cache[mycp] = cpv_list + return cpv_list[:] def cp_all(self): return list(self.cpdict) diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py index 9f562820b..940aa1020 100644 --- a/pym/portage/sets/__init__.py +++ b/pym/portage/sets/__init__.py @@ -94,14 +94,17 @@ class SetConfig(SafeConfigParser): self.aliases = shortnames return self.aliases - def getSetAtoms(self, setname, ignorelist=[]): + def getSetAtoms(self, setname, ignorelist=None): myset = self.getSetsWithAliases()[setname] myatoms = myset.getAtoms() - ignorelist.append(setname) + if ignorelist is None: + ignorelist = set() + ignorelist.add(setname) for n in myset.getNonAtoms(): if n[0] == SETPREFIX and n[1:] in self.aliases: if n[1:] not in ignorelist: - myatoms.update(self.getSetAtoms(n[1:])) + myatoms.update(self.getSetAtoms(n[1:], + ignorelist=ignorelist)) return myatoms def make_default_config(settings, trees): diff --git a/pym/portage/sets/base.py b/pym/portage/sets/base.py index c22ee9953..8e5863cba 100644 --- a/pym/portage/sets/base.py +++ b/pym/portage/sets/base.py @@ -26,7 +26,8 @@ class PackageSet(object): self._nonatoms = set() def __contains__(self, atom): - return atom in self.getAtoms() or atom in self._nonatoms + self._load() + return atom in self._atoms or atom in self._nonatoms def __iter__(self): for x in self.getAtoms(): @@ -36,18 +37,21 @@ class PackageSet(object): if not op in OPERATIONS: raise ValueError(op) return op in self._operations - - def getAtoms(self): + + def _load(self): if not (self._loaded or self._loading): self._loading = True self.load() self._loaded = True self._loading = False - return self._atoms + + def getAtoms(self): + self._load() + return self._atoms.copy() def getNonAtoms(self): - self.getAtoms() - return self._nonatoms + self._load() + return self._nonatoms.copy() def _setAtoms(self, atoms): atoms = map(str.strip, atoms)