From 6e2caf7f736d0babedfb59d9e60a67ab18869292 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 21 Sep 2009 17:41:37 +0000 Subject: [PATCH] Merged from trunk -r14296:14306 | 14297 | Add back a line which I erroneously removed when committing | | zmedico | the patch for bug 203733. Thanks to Alexis Ballier | | | for reporting the related KeyError. | | 14298 | In doebuild_environment(), detect if setcpv() hasn't been | | zmedico | called properly, and call it if necessary (mainly for api | | | consumers). | | 14299 | Make _unicode_module_wrapper cache wrappers and reuse them. | | zmedico | Thanks to Marat Radchenko for this | | | patch from bug #276813. | | 14300 | Make Atom inherit from str. Thanks to Marat Radchenko | | zmedico | for this patch from bug #276813. | | 14301 | Use _unicode_encode() to ensure that only unicode strings | | zmedico | are passed into io.StringIO text streams. | | 14302 | Misc unicode fixes for repoman. | | zmedico | | | 14303 | Add support in ConsoleStyleFile.write() for unicode encoding | | zmedico | when writing to stdout/stderr in python-2.x. | | 14304 | Add fallback in ImportError for io.StringIO. This is needed | | zmedico | for python-2.6 with USE=build since io imports threading | | | which imports thread which is unavailable. | | 14305 | Fix depgraph._validate_blockers() so that it doesn't do | | zmedico | prohibited Atom(Atom) constructor calls. | | 14306 | Fix typo from previous commit. | | zmedico | | svn path=/main/branches/prefix/; revision=14342 --- pym/_emerge/JobStatusDisplay.py | 37 +++++++----- pym/_emerge/depgraph.py | 9 ++- pym/portage/__init__.py | 28 +++++++-- pym/portage/cvstree.py | 1 + pym/portage/dep.py | 103 ++++++++------------------------ pym/portage/output.py | 32 ++++++++-- pym/portage/util.py | 8 ++- pym/repoman/utilities.py | 19 +++--- 8 files changed, 120 insertions(+), 117 deletions(-) diff --git a/pym/_emerge/JobStatusDisplay.py b/pym/_emerge/JobStatusDisplay.py index dd72a2971..f543d00bb 100644 --- a/pym/_emerge/JobStatusDisplay.py +++ b/pym/_emerge/JobStatusDisplay.py @@ -5,11 +5,18 @@ import formatter import sys import time -from io import StringIO +try: + from io import StringIO +except ImportError: + # Needed for python-2.6 with USE=build since + # io imports threading which imports thread + # which is unavailable. + from StringIO import StringIO import portage from portage import os from portage import _encodings +from portage import _unicode_decode from portage.output import xtermTitle from _emerge.getloadavg import getloadavg @@ -222,36 +229,36 @@ class JobStatusDisplay(object): f = formatter.AbstractFormatter(style_writer) number_style = "INFORM" - f.add_literal_data("Jobs: ") + f.add_literal_data(_unicode_decode("Jobs: ")) f.push_style(number_style) - f.add_literal_data(curval_str) + f.add_literal_data(_unicode_decode(curval_str)) f.pop_style() - f.add_literal_data(" of ") + f.add_literal_data(_unicode_decode(" of ")) f.push_style(number_style) - f.add_literal_data(maxval_str) + f.add_literal_data(_unicode_decode(maxval_str)) f.pop_style() - f.add_literal_data(" complete") + f.add_literal_data(_unicode_decode(" complete")) if self.running: - f.add_literal_data(", ") + f.add_literal_data(_unicode_decode(", ")) f.push_style(number_style) - f.add_literal_data(running_str) + f.add_literal_data(_unicode_decode(running_str)) f.pop_style() - f.add_literal_data(" running") + f.add_literal_data(_unicode_decode(" running")) if self.failed: - f.add_literal_data(", ") + f.add_literal_data(_unicode_decode(", ")) f.push_style(number_style) - f.add_literal_data(failed_str) + f.add_literal_data(_unicode_decode(failed_str)) f.pop_style() - f.add_literal_data(" failed") + f.add_literal_data(_unicode_decode(" failed")) padding = self._jobs_column_width - len(plain_output.getvalue()) if padding > 0: - f.add_literal_data(padding * " ") + f.add_literal_data(padding * _unicode_decode(" ")) - f.add_literal_data("Load avg: ") - f.add_literal_data(load_avg_str) + f.add_literal_data(_unicode_decode("Load avg: ")) + f.add_literal_data(_unicode_decode(load_avg_str)) # Truncate to fit width, to avoid making the terminal scroll if the # line overflows (happens when the load average is large). diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 1d5f38083..ab342929e 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -2604,8 +2604,7 @@ class depgraph(object): except KeyError: pass if blockers is not None: - blockers = set(str(blocker.atom) \ - for blocker in blockers) + blockers = set(blocker.atom for blocker in blockers) # If this node has any blockers, create a "nomerge" # node for it so that they can be enforced. @@ -2636,7 +2635,7 @@ class depgraph(object): continue if blocker_data: - blocker_atoms = blocker_data.atoms + blocker_atoms = [Atom(atom) for atom in blocker_data.atoms] else: # Use aux_get() to trigger FakeVartree global # updates on *DEPEND when appropriate. @@ -2675,7 +2674,7 @@ class depgraph(object): show_invalid_depstring_notice(pkg, depstr, atoms) return False blocker_atoms = [myatom for myatom in atoms \ - if myatom.startswith("!")] + if myatom.blocker] blocker_atoms.sort() counter = long(pkg.metadata["COUNTER"]) blocker_cache[cpv] = \ @@ -2683,7 +2682,7 @@ class depgraph(object): if blocker_atoms: try: for atom in blocker_atoms: - blocker = Blocker(atom=portage.dep.Atom(atom), + blocker = Blocker(atom=atom, eapi=pkg.metadata["EAPI"], root=myroot) self._dynamic_config._blocker_parents.add(blocker, pkg) except portage.exception.InvalidAtom as e: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index b95205945..191e5385d 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -211,14 +211,24 @@ class _unicode_module_wrapper(object): """ Wraps a module and wraps all functions with _unicode_func_wrapper. """ - __slots__ = ('_mod', '_encoding', '_overrides') + __slots__ = ('_mod', '_encoding', '_overrides', '_cache') - def __init__(self, mod, encoding=_encodings['fs'], overrides=None): + def __init__(self, mod, encoding=_encodings['fs'], overrides=None, cache=True): object.__setattr__(self, '_mod', mod) object.__setattr__(self, '_encoding', encoding) object.__setattr__(self, '_overrides', overrides) + if cache: + cache = {} + else: + cache = None + object.__setattr__(self, '_cache', cache) def __getattribute__(self, attr): + cache = object.__getattribute__(self, '_cache') + if cache is not None: + result = cache.get(attr) + if result is not None: + return result result = getattr(object.__getattribute__(self, '_mod'), attr) encoding = object.__getattribute__(self, '_encoding') overrides = object.__getattribute__(self, '_overrides') @@ -234,6 +244,8 @@ class _unicode_module_wrapper(object): encoding=encoding, overrides=overrides) elif hasattr(result, '__call__'): result = _unicode_func_wrapper(result, encoding=encoding) + if cache is not None: + cache[attr] = result return result import os as _os @@ -5903,12 +5915,18 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m # so that the caller can override it. tmpdir = mysettings["PORTAGE_TMPDIR"] - if mycpv != mysettings.mycpv: - if mydo == 'depend': + if mydo == 'depend': + if mycpv != mysettings.mycpv: # Don't pass in mydbapi here since the resulting aux_get # call would lead to infinite 'depend' phase recursion. mysettings.setcpv(mycpv) - else: + else: + # If IUSE isn't in configdict['pkg'], it means that setcpv() + # hasn't been called with the mydb argument, so we have to + # call it here (portage code always calls setcpv properly, + # but api consumers might not). + if mycpv != mysettings.mycpv or \ + 'IUSE' not in mysettings.configdict['pkg']: # Reload env.d variables and reset any previous settings. mysettings.reload() mysettings.reset() diff --git a/pym/portage/cvstree.py b/pym/portage/cvstree.py index 1ecffa493..0dd8f10bc 100644 --- a/pym/portage/cvstree.py +++ b/pym/portage/cvstree.py @@ -254,6 +254,7 @@ def getentries(mydir,recursive=0): print(mydir,file) if os.path.isdir(mydir+"/"+file): if file not in entries["dirs"]: + entries["dirs"][file]={"dirs":{},"files":{}} # It's normal for a directory to be unlisted in Entries # when checked out without -P (see bug #257660). rentries=getentries(mydir+"/"+file,recursive) diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 66557c0c6..68e6c6b22 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -488,16 +488,18 @@ class _use_dep(object): return _use_dep(tokens) -class Atom(object): +if sys.hexversion < 0x3000000: + _atom_base = unicode +else: + _atom_base = str + +class Atom(_atom_base): """ For compatibility with existing atom string manipulation code, this class emulates most of the str methods that are useful with atoms. """ - __slots__ = ("__weakref__", "blocker", "cp", "cpv", "operator", - "slot", "use", "without_use", "_str",) - class _blocker(object): __slots__ = ("overlap",) @@ -510,10 +512,12 @@ class Atom(object): def __init__(self, forbid_overlap=False): self.overlap = self._overlap(forbid=forbid_overlap) - def __init__(self, mypkg): - s = mypkg = str(mypkg) - obj_setattr = object.__setattr__ - obj_setattr(self, '_str', s) + def __init__(self, s): + if isinstance(s, Atom): + # This is an efficiency assertion, to ensure that the Atom + # constructor is not called redundantly. + raise TypeError(_("Expected %s, got %s") % \ + (_atom_base, type(s))) if "!" == s[:1]: blocker = self._blocker(forbid_overlap=("!" == s[1:2])) @@ -523,10 +527,10 @@ class Atom(object): s = s[1:] else: blocker = False - obj_setattr(self, "blocker", blocker) + self.__dict__['blocker'] = blocker m = _atom_re.match(s) if m is None: - raise InvalidAtom(mypkg) + raise InvalidAtom(self) if m.group('op') is not None: base = _atom_re.groupindex['op'] @@ -534,25 +538,25 @@ class Atom(object): cpv = m.group(base + 2) cp = m.group(base + 3) if m.group(base + 4) is not None: - raise InvalidAtom(mypkg) + raise InvalidAtom(self) elif m.group('star') is not None: base = _atom_re.groupindex['star'] op = '=*' cpv = m.group(base + 1) cp = m.group(base + 2) if m.group(base + 3) is not None: - raise InvalidAtom(mypkg) + raise InvalidAtom(self) elif m.group('simple') is not None: op = None cpv = cp = m.group(_atom_re.groupindex['simple'] + 1) if m.group(_atom_re.groupindex['simple'] + 2) is not None: - raise InvalidAtom(mypkg) + raise InvalidAtom(self) else: - raise AssertionError(_("required group not found in atom: '%s'") % s) - obj_setattr(self, "cp", cp) - obj_setattr(self, "cpv", cpv) - obj_setattr(self, "slot", m.group(_atom_re.groups - 1)) - obj_setattr(self, "operator", op) + raise AssertionError(_("required group not found in atom: '%s'") % self) + self.__dict__['cp'] = cp + self.__dict__['cpv'] = cpv + self.__dict__['slot'] = m.group(_atom_re.groups - 1) + self.__dict__['operator'] = op use_str = m.group(_atom_re.groups) if use_str is not None: @@ -562,8 +566,8 @@ class Atom(object): use = None without_use = self - obj_setattr(self, "use", use) - obj_setattr(self, "without_use", without_use) + self.__dict__['use'] = use + self.__dict__['without_use'] = without_use def __setattr__(self, name, value): raise AttributeError("Atom instances are immutable", @@ -600,65 +604,6 @@ class Atom(object): return False - # Implement some common str methods. - - def __eq__(self, other): - return self._str == other - - def __getitem__(self, key): - return self._str[key] - - def __hash__(self): - return hash(self._str) - - def __len__(self): - return len(self._str) - - def __lt__(self, other): - return self._str < other - - def __ne__(self, other): - return self._str != other - - def __repr__(self): - return repr(self._str) - - def __str__(self): - return self._str - - def endswith(self, *pargs, **kargs): - return self._str.endswith(*pargs, **kargs) - - def find(self, *pargs, **kargs): - return self._str.find(*pargs, **kargs) - - def index(self, *pargs, **kargs): - return self._str.index(*pargs, **kargs) - - def lstrip(self, *pargs, **kargs): - return self._str.lstrip(*pargs, **kargs) - - def replace(self, *pargs, **kargs): - return self._str.replace(*pargs, **kargs) - - def startswith(self, *pargs, **kargs): - return self._str.startswith(*pargs, **kargs) - - def split(self, *pargs, **kargs): - return self._str.split(*pargs, **kargs) - - def strip(self, *pargs, **kargs): - return self._str.strip(*pargs, **kargs) - - def rindex(self, *pargs, **kargs): - return self._str.rindex(*pargs, **kargs) - - def rfind(self, *pargs, **kargs): - return self._str.rfind(*pargs, **kargs) - - def rstrip(self, *pargs, **kargs): - return self._str.rstrip(*pargs, **kargs) - def __copy__(self): """Immutable, so returns self.""" return self diff --git a/pym/portage/output.py b/pym/portage/output.py index 589456059..f946d47ac 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -22,6 +22,7 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import os from portage import _encodings from portage import _unicode_encode +from portage import _unicode_decode from portage.const import COLOR_MAP_FILE, EPREFIX from portage.exception import CommandNotFound, FileNotFound, \ ParseError, PermissionDenied, PortageException @@ -354,16 +355,31 @@ class ConsoleStyleFile(object): self._styles = styles def write(self, s): + # In python-2.6, DumbWriter.send_line_break() can write + # non-unicode '\n' which fails with TypeError if self._file + # is a text stream such as io.StringIO. Therefore, make sure + # input is converted to unicode when necessary. + s = _unicode_decode(s) global havecolor if havecolor and self._styles: + styled_s = [] for style in self._styles: - self._file.write(style_to_ansi_code(style)) - self._file.write(s) - self._file.write(codes["reset"]) + styled_s.append(style_to_ansi_code(style)) + styled_s.append(s) + styled_s.append(codes["reset"]) + self._write(self._file, "".join(styled_s)) else: - self._file.write(s) + self._write(self._file, s) if self.write_listener: - self.write_listener.write(s) + self._write(self.write_listener, s) + + def _write(self, f, s): + if sys.hexversion < 0x3000000 and \ + isinstance(s, unicode) and \ + f in (sys.stdout, sys.stderr): + # avoid potential UnicodeEncodeError + s = s.encode(_encodings['stdio'], 'backslashreplace') + f.write(s) def writelines(self, lines): for s in lines: @@ -743,6 +759,12 @@ def _init(config_root='/'): codes = object.__getattribute__(codes, '_attr') _styles = object.__getattribute__(_styles, '_attr') + for k, v in codes.iteritems(): + codes[k] = _unicode_decode(v) + + for k, v in _styles.iteritems(): + _styles[k] = _unicode_decode(v) + try: _parse_color_map(config_root=config_root, onerror=lambda e: writemsg("%s\n" % str(e), noiselevel=-1)) diff --git a/pym/portage/util.py b/pym/portage/util.py index 021c308f0..c743ca774 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -24,7 +24,13 @@ import shlex import stat import string import sys -from io import StringIO +try: + from io import StringIO +except ImportError: + # Needed for python-2.6 with USE=build since + # io imports threading which imports thread + # which is unavailable. + from StringIO import StringIO import portage from portage import os diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py index fcabd599d..3d8d43aed 100644 --- a/pym/repoman/utilities.py +++ b/pym/repoman/utilities.py @@ -33,6 +33,7 @@ from xml.dom import NotFoundErr from xml.parsers.expat import ExpatError from portage import os from portage import _encodings +from portage import _unicode_decode from portage import _unicode_encode from portage import output from portage.output import red, green @@ -221,12 +222,12 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning # we only want key value pairs where value > 0 for category, number in \ itertools.ifilter(lambda myitem: myitem[1] > 0, stats.iteritems()): - formatter.add_literal_data(" " + category.ljust(30)) + formatter.add_literal_data(_unicode_decode(" " + category.ljust(30))) if category in qawarnings: formatter.push_style("WARN") else: formatter.push_style("BAD") - formatter.add_literal_data(str(number)) + formatter.add_literal_data(_unicode_decode(str(number))) formatter.pop_style() formatter.add_line_break() if not dofull: @@ -237,7 +238,7 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning if not full and len(fails_list) > 12: fails_list = fails_list[:12] for failure in fails_list: - formatter.add_literal_data(" " + failure) + formatter.add_literal_data(_unicode_decode(" " + failure)) formatter.add_line_break() @@ -276,13 +277,17 @@ def get_commit_message_with_editor(editor, message=None): from tempfile import mkstemp fd, filename = mkstemp() try: - os.write(fd, "\n# Please enter the commit message " + \ + os.write(fd, _unicode_encode( + "\n# Please enter the commit message " + \ "for your changes.\n# (Comment lines starting " + \ - "with '#' will not be included)\n") + "with '#' will not be included)\n", + encoding=_encodings['content'], errors='backslashreplace')) if message: - os.write(fd, "#\n") + os.write(fd, _unicode_encode("#\n", + encoding=_encodings['content'], errors='backslashreplace')) for line in message: - os.write(fd, "#" + line) + os.write(fd, _unicode_encode("#" + line, + encoding=_encodings['content'], errors='backslashreplace')) os.close(fd) retval = os.system(editor + " '%s'" % filename) if not (os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == os.EX_OK): -- 2.26.2