From: Fabian Groffen Date: Sun, 27 May 2007 21:09:50 +0000 (-0000) Subject: Merged from trunk 6634:6644 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=73fd192603de48af514a5d666b9253f36d44814f;p=portage.git Merged from trunk 6634:6644 +------------------------------------------------------------------------+ | 6634 | Fix broken dict.get logic. | | zmedico | | |---------+--------------------------------------------------------------| | 6635 | Fix logic for USE_EXPAND variables that are incremental | | zmedico | (there are none currently). | |---------+--------------------------------------------------------------| | 6637 | In the WorldHandler class, add support for an onProgress | | zmedico | callback that can be used notify the user of job progress. | | | This isn't really useful for WorldHandler since it completes | | | quickly, but it will be useful for longer running tasks. | |---------+--------------------------------------------------------------| | 6638 | Add a get_term_size() function that uses the curses module | | zmedico | if available and otherwise falls back to parsing the output | | | of `stty size`. | |---------+--------------------------------------------------------------| | 6639 | Add progress support to emaint (similar to wget's progress | | zmedico | bar). | |---------+--------------------------------------------------------------| | 6640 | Add SIGWINCH support so that the progress bar is resized if | | zmedico | the xterm is resized. | |---------+--------------------------------------------------------------| | 6641 | Fix an UnboundLocalError when there's no tty. | | zmedico | | |---------+--------------------------------------------------------------| | 6642 | Fix a typo. | | zmedico | | |---------+--------------------------------------------------------------| | 6643 | Add an emaint "binhost" target which does the time consuming | | zmedico | population of $PKGDIR/Packages that is necessary for a | | | binhost (MD5 makes it relatively slow). It reuses metadata | | | from the existing Packages file whenever possible. | |---------+--------------------------------------------------------------| | 6644 | Make emaint prune stale entires from $PKGDIR/Packages. | | zmedico | | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=6646 --- diff --git a/bin/emaint b/bin/emaint index 1b8593459..17e6ef15a 100755 --- a/bin/emaint +++ b/bin/emaint @@ -1,6 +1,6 @@ #!@PYTHON@ -O -import sys, os +import sys, os, time, signal from optparse import OptionParser, OptionValueError if not hasattr(__builtins__, "set"): from sets import Set as set @@ -12,7 +12,7 @@ except ImportError: sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage -import portage.const, portage.exception +import portage.const, portage.exception, portage.output class WorldHandler(object): def name(): @@ -27,13 +27,20 @@ class WorldHandler(object): self.world_file = os.path.join("/", portage.const.WORLD_FILE) self.found = os.access(self.world_file, os.R_OK) + def _check_world(self, onProgress): categories = set(portage.settings.categories) myroot = portage.settings["ROOT"] vardb = portage.db[myroot]["vartree"].dbapi - for atom in open(self.world_file).read().split(): + world_atoms = open(self.world_file).read().split() + maxval = len(world_atoms) + if onProgress: + onProgress(maxval, 0) + for i, atom in enumerate(world_atoms): if not portage.isvalidatom(atom): self.invalid.append(atom) + if onProgress: + onProgress(maxval, i+1) continue okay = True if not vardb.match(atom): @@ -44,8 +51,11 @@ class WorldHandler(object): okay = False if okay: self.okay.append(atom) + if onProgress: + onProgress(maxval, i+1) - def check(self): + def check(self, onProgress=None): + self._check_world(onProgress) errors = [] if self.found: errors += map(lambda x: "'%s' is not a valid atom" % x, self.invalid) @@ -55,7 +65,8 @@ class WorldHandler(object): errors.append(self.world_file + " could not be opened for reading") return errors - def fix(self): + def fix(self, onProgress=None): + self._check_world(onProgress) errors = [] try: portage.write_atomic(self.world_file, "\n".join(self.okay)) @@ -63,6 +74,95 @@ class WorldHandler(object): errors.append(self.world_file + " could not be opened for writing") return errors +class BinhostHandler(object): + + def name(): + return "binhost" + name = staticmethod(name) + + def __init__(self): + myroot = portage.settings["ROOT"] + self._bintree = portage.db[myroot]["bintree"] + self._bintree.populate() + self._pkgindex_file = os.path.join(self._bintree.pkgdir, "Packages") + from portage import getbinpkg + self._pkgindex = getbinpkg.PackageIndex() + f = open(self._pkgindex_file, 'r') + try: + self._pkgindex.read(f) + finally: + f.close() + + def check(self, onProgress=None): + missing = [] + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + for i, cpv in enumerate(cpv_all): + d = pkgindex.packages.get(cpv) + if not d or "MD5" not in d: + missing.append(cpv) + if onProgress: + onProgress(maxval, i+1) + errors = ["'%s' is not in Packages" % cpv for cpv in missing] + stale = set(pkgindex.packages).difference(cpv_all) + for cpv in stale: + errors.append("'%s' is not in the repository" % cpv) + return errors + + def fix(self, onProgress=None): + bintree = self._bintree + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + missing = [] + maxval = len(cpv_all) + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + for i, cpv in enumerate(cpv_all): + d = pkgindex.packages.get(cpv) + if not d or "MD5" not in d: + bintree.inject(cpv) + if onProgress: + onProgress(maxval, i+1) + stale = set(pkgindex.packages).difference(cpv_all) + if stale: + from portage import locks + pkgindex_lock = locks.lockfile( + self._pkgindex_file, wantnewlockfile=1) + try: + from portage import getbinpkg + del pkgindex + self._pkgindex = getbinpkg.PackageIndex() + f = open(self._pkgindex_file, 'r') + try: + self._pkgindex.read(f) + finally: + f.close() + from portage.dbapi.bintree import binarytree + self._bintree = binarytree(bintree.root, bintree.pkgdir, + settings=bintree.settings) + del bintree + portage.db[self._bintree.root]["bintree"] = self._bintree + self._bintree._populate() + for cpv in set(self._pkgindex.packages).difference( + self._bintree.dbapi.cpv_all()): + del self._pkgindex.packages[cpv] + from portage.util import atomic_ofstream + f = atomic_ofstream(self._pkgindex_file) + try: + self._pkgindex.write(f) + finally: + f.close() + finally: + locks.unlockfile(pkgindex_lock) + return None + class VdbKeyHandler(object): def name(): return "vdbkeys" @@ -121,11 +221,32 @@ class VdbKeyHandler(object): return errors +class ProgressHandler(object): + def __init__(self): + self.curval = 0 + self.maxval = 0 + self.last_update = 0 + self.min_display_latency = 0.2 + + def onProgress(self, maxval, curval): + self.maxval = maxval + self.curval = curval + cur_time = time.time() + if cur_time - self.last_update >= self.min_display_latency: + self.last_update = cur_time + self.display() + + def display(self): + raise NotImplementedError(self) + def emaint_main(myargv): # TODO: Create a system that allows external modules to be added without # the need for hard coding. - modules = {"world" : WorldHandler} + modules = { + "world" : WorldHandler, + "binhost":BinhostHandler + } module_names = modules.keys() module_names.sort() @@ -180,11 +301,28 @@ def emaint_main(myargv): status = "Attempting to fix %s" func = "fix" - + isatty = sys.stdout.isatty() for task in tasks: print status % task.name() inst = task() - result = getattr(inst, func)() + onProgress = None + if isatty: + progressBar = portage.output.TermProgressBar() + progressHandler = ProgressHandler() + onProgress = progressHandler.onProgress + def display(): + progressBar.set(progressHandler.curval, progressHandler.maxval) + progressHandler.display = display + def sigwinch_handler(signum, frame): + lines, progressBar.term_columns = \ + portage.output.get_term_size() + signal.signal(signal.SIGWINCH, sigwinch_handler) + result = getattr(inst, func)(onProgress=onProgress) + if isatty: + # make sure the final progress is displayed + progressHandler.display() + print + signal.signal(signal.SIGWINCH, signal.SIG_DFL) if result: print print "\n".join(result) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 8934248d8..c41248302 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1977,23 +1977,31 @@ class config: for var in cur_use_expand: var_lower = var.lower() - if var not in myincrementals: + is_not_incremental = var not in myincrementals + if is_not_incremental: prefix = var_lower + "_" for x in list(myflags): if x.startswith(prefix): myflags.remove(x) for x in curdb[var].split(): - # Any incremental USE_EXPAND variables have already been - # processed, so leading +/- operators are invalid here. if x[0] == "+": - writemsg(colorize("BAD", "Invalid '+' operator in " + \ - "non-incremental variable '%s': '%s'\n" % (var, x)), - noiselevel=-1) + if is_not_incremental: + writemsg(colorize("BAD", "Invalid '+' " + \ + "operator in non-incremental variable " + \ + "'%s': '%s'\n" % (var, x)), noiselevel=-1) + continue + else: + writemsg(colorize("BAD", "Invalid '+' " + \ + "operator in incremental variable " + \ + "'%s': '%s'\n" % (var, x)), noiselevel=-1) x = x[1:] if x[0] == "-": - writemsg(colorize("BAD", "Invalid '-' operator in " + \ - "non-incremental variable '%s': '%s'\n" % (var, x)), - noiselevel=-1) + if is_not_incremental: + writemsg(colorize("BAD", "Invalid '-' " + \ + "operator in non-incremental variable " + \ + "'%s': '%s'\n" % (var, x)), noiselevel=-1) + continue + myflags.discard(var_lower + "_" + x[1:]) continue myflags.add(var_lower + "_" + x) diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index c81de292e..8ef1ff23b 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -361,7 +361,7 @@ class binarytree(object): if pkgindex_lock: unlockfile(pkgindex_lock) - def _populate(self, getbinpkgs, getbinpkgsonly): + def _populate(self, getbinpkgs=0, getbinpkgsonly=0): if (not os.path.isdir(self.pkgdir) and not getbinpkgs): return 0 diff --git a/pym/portage/output.py b/pym/portage/output.py index 406f02b53..b46a00ed2 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -4,7 +4,7 @@ __docformat__ = "epytext" -import commands,errno,os,re,shlex,sys +import commands, errno, os, re, shlex, sys, time from portage.const import COLOR_MAP_FILE from portage.util import writemsg from portage.exception import PortageException, ParseError, PermissionDenied, FileNotFound @@ -237,6 +237,29 @@ def create_color_func(color_key): for c in compat_functions_colors: globals()[c] = create_color_func(c) +def get_term_size(): + """ + Get the number of lines and columns of the tty that is connected to + stdout. Returns a tuple of (lines, columns) or (-1, -1) if an error + occurs. The curses module is used if available, otherwise the output of + `stty size` is parsed. + """ + try: + import curses + curses.setupterm() + return curses.tigetnum('lines'), curses.tigetnum('cols') + except ImportError: + pass + st, out = commands.getstatusoutput('stty size') + if st == os.EX_OK: + out = out.split() + if len(out) == 2: + try: + return int(out[0]), int(out[1]) + except ValueError: + pass + return -1, -1 + class EOutput: """ Performs fancy terminal formatting for status and informational messages. @@ -264,17 +287,7 @@ class EOutput: self.__last_e_cmd = "" self.__last_e_len = 0 self.quiet = False - columns = 0 - try: - columns = int(os.getenv("COLUMNS", 0)) - except ValueError: - pass - if columns <= 0: - try: - columns = int(commands.getoutput( - 'set -- `stty size 2>/dev/null` ; echo "$2"')) - except ValueError: - pass + lines, columns = get_term_size() if columns <= 0: columns = 80 # Adjust columns so that eend works properly on a standard BSD console. @@ -394,3 +407,129 @@ class EOutput: if not self.quiet: self.__eend("ewend", errno, msg) self.__last_e_cmd = "ewend" + +class ProgressBar(object): + """The interface is copied from the ProgressBar class from the EasyDialogs + module (which is Mac only).""" + def __init__(self, title=None, maxval=0, label=None): + self._title = title + self._maxval = maxval + self._label = maxval + self._curval = 0 + + @property + def curval(self): + """ + The current value (of type integer or long integer) of the progress + bar. The normal access methods coerce curval between 0 and maxval. This + attribute should not be altered directly. + """ + return self._curval + + @property + def maxval(self): + """ + The maximum value (of type integer or long integer) of the progress + bar; the progress bar (thermometer style) is full when curval equals + maxval. If maxval is 0, the bar will be indeterminate (barber-pole). + This attribute should not be altered directly. + """ + return self._maxval + + def title(self, newstr): + """Sets the text in the title bar of the progress dialog to newstr.""" + self._title = newstr + + def label(self, newstr): + """Sets the text in the progress box of the progress dialog to newstr.""" + self._label = newstr + + def set(self, value, maxval=None): + """ + Sets the progress bar's curval to value, and also maxval to max if the + latter is provided. value is first coerced between 0 and maxval. The + thermometer bar is updated to reflect the changes, including a change + from indeterminate to determinate or vice versa. + """ + if maxval is not None: + self._maxval = maxval + if value < 0: + value = 0 + elif value > maxval: + value = maxval + self._curval = value + + def inc(self, n=1): + """Increments the progress bar's curval by n, or by 1 if n is not + provided. (Note that n may be negative, in which case the effect is a + decrement.) The progress bar is updated to reflect the change. If the + bar is indeterminate, this causes one ``spin'' of the barber pole. The + resulting curval is coerced between 0 and maxval if incrementing causes + it to fall outside this range. + """ + self.set(self._curval+n) + +class TermProgressBar(ProgressBar): + """A tty progress bar similar to wget's.""" + def __init__(self, **kwargs): + ProgressBar.__init__(self, **kwargs) + lines, self.term_columns = get_term_size() + self.file = sys.stdout + self._min_columns = 11 + # for indeterminate mode, ranges from 0.0 to 1.0 + self._position = 0.0 + + def set(self, value, maxval=None): + ProgressBar.set(self, value, maxval=maxval) + self._display_image(self._create_image()) + + def _display_image(self, image): + self.file.write('\r') + self.file.write(image) + self.file.flush() + + def _create_image(self): + cols = self.term_columns + min_columns = self._min_columns + curval = self._curval + maxval = self._maxval + position = self._position + if cols < 3: + return "" + bar_space = cols - 6 + if maxval == 0: + max_bar_width = bar_space-3 + image = " " + if cols < min_columns: + return image + if position <= 0.5: + offset = 2 * position + else: + offset = 2 * (1 - position) + delta = 0.5 / max_bar_width + position += delta + if position >= 1.0: + position = 0.0 + # make sure it touches the ends + if 1.0 - position < delta: + position = 1.0 + if position < 0.5 and 0.5 - position < delta: + position = 0.5 + self._position = position + bar_width = int(offset * max_bar_width) + image = image + "[" + (bar_width * " ") + \ + "<=>" + ((max_bar_width - bar_width) * " ") + "]" + return image + else: + max_bar_width = bar_space-1 + percentage = int(100 * float(curval) / maxval) + if percentage == 100: + percentage = 99 + image = ("%d%% " % percentage).rjust(4) + if cols < min_columns: + return image + offset = float(curval) / maxval + bar_width = int(offset * max_bar_width) + image = image + "[" + (bar_width * "=") + \ + ">" + ((max_bar_width - bar_width) * " ") + "]" + return image