From d288cbda498f8c51d38cb793ca9bf8082816ed31 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 23 Feb 2009 19:34:06 +0000 Subject: [PATCH] Merged from trunk -r12686:12697 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit | 12687 | When decoding term codes inside the JobStatusDisplay | | zmedico | constructor, use the 'replace' error handling scheme in | | | order to avoid triggering a potential UnicodeError. | | 12688 | Add support in _ensure_default_encoding(), add support for | | zmedico | utf_8 and fall back to utf_8 if the encoding returned by | | | sys.getdefaultencoding() is unavailable. | | 12689 | When encoding strings inside vardbapi._counter_hash() and | | zmedico | vardbapi._owners_cache._hash_str(), use 'backslashreplace' | | | in order to avoid potential UnicodeError exceptions. Use | | | constant ascii and utf_8 codecs, respectively, since these | | | codecs are guaranteed to be available by | | | _ensure_default_encoding(). | | 12690 | Bump vardbapi._owners_cache_version from 1 to 2 since hash | | zmedico | value encoding is now hardcoded as utf_8. | | 12691 | When displaying reverse dependencies in verbose | | zmedico | --depclean/--prune output, sort the parent packages by cpv. | | 12692 | Don't try to call setdefaultencoding() since it's not | | zmedico | intended to be called. | | 12693 | Revert vardbapi._owners_cache_version and unicode handling | | zmedico | since it needs some more work. | | 12694 | Reorganize _ensure_default_encoding(). | | zmedico | | | 12695 | Fix default_encoding logic in _ensure_default_encoding(). | | zmedico | | | 12696 | Fix getmaskingstatus() so that it doesn't inappropriately | | zmedico | return a 'missing keyword' mask is some cases when the user | | | has ** in package.keywords. Thanks to Raúl Porcel | | | for reporting. | | 12697 | In config.setcpv(), increase efficiency by filtering USE | | zmedico | with IUSE before calculating USE_EXPAND variables. Also use | | | a generator expression instead of a list comprehension in | | | the USE_EXPAND generation code. | svn path=/main/branches/prefix/; revision=12699 --- pym/_emerge/__init__.py | 16 +++- pym/portage/__init__.py | 154 +++++++++++++++++++++++++---------- pym/portage/dbapi/vartree.py | 6 +- 3 files changed, 128 insertions(+), 48 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 8fc5ef24d..beb4c710d 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -9679,9 +9679,10 @@ class JobStatusDisplay(object): for k, capname in self._termcap_name_map.iteritems(): term_codes[k] = self._default_term_codes[capname] object.__setattr__(self, "_term_codes", term_codes) + encoding = sys.getdefaultencoding() for k, v in self._term_codes.items(): if not isinstance(v, str): - self._term_codes[k] = v.decode() + self._term_codes[k] = v.decode(encoding, 'replace') def _init_term(self): """ @@ -13718,13 +13719,22 @@ def action_depclean(settings, trees, ldpath_mtimes, msg.append("\n") portage.writemsg_stdout("".join(msg), noiselevel=-1) + def cmp_pkg_cpv(pkg1, pkg2): + """Sort Package instances by cpv.""" + if pkg1.cpv > pkg2.cpv: + return 1 + elif pkg1.cpv == pkg2.cpv: + return 0 + else: + return -1 + def create_cleanlist(): pkgs_to_remove = [] if action == "depclean": if args_set: - for pkg in vardb: + for pkg in sorted(vardb, key=cmp_sort_key(cmp_pkg_cpv)): arg_atom = None try: arg_atom = args_set.findAtomForPackage(pkg) @@ -13739,7 +13749,7 @@ def action_depclean(settings, trees, ldpath_mtimes, show_parents(pkg) else: - for pkg in vardb: + for pkg in sorted(vardb, key=cmp_sort_key(cmp_pkg_cpv)): if pkg not in graph: pkgs_to_remove.append(pkg) elif "--verbose" in myopts: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 802525815..b760d8423 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -12,6 +12,7 @@ VERSION="$Rev$"[6:-2] + "-svn" try: import sys + import codecs import copy import errno import logging @@ -2133,6 +2134,8 @@ class config(object): if ebuild_force_test: self.usemask.discard("test") + use.difference_update([x for x in use if x not in iuse_implicit]) + # Use the calculated USE flags to regenerate the USE_EXPAND flags so # that they are consistent. For optimal performance, use slice # comparison instead of startswith(). @@ -2140,8 +2143,8 @@ class config(object): for var in use_expand: prefix = var.lower() + "_" prefix_len = len(prefix) - expand_flags = set([ x[prefix_len:] for x in use \ - if x[:prefix_len] == prefix ]) + expand_flags = set( x[prefix_len:] for x in use \ + if x[:prefix_len] == prefix ) var_split = self.get(var, "").split() # Preserve the order of var_split because it can matter for things # like LINGUAS. @@ -2203,9 +2206,7 @@ class config(object): # attribute since we still want to be able to see global USE # settings for things like emerge --info. - self.configdict["pkg"]["PORTAGE_USE"] = " ".join(sorted( - x for x in use if \ - x in iuse_implicit)) + self.configdict["pkg"]["PORTAGE_USE"] = " ".join(sorted(use)) def _get_implicit_iuse(self): """ @@ -7250,9 +7251,13 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): kmask = "missing" - for keyword in pgroups: - if keyword in mygroups: - kmask=None + if '**' in pgroups: + kmask = None + else: + for keyword in pgroups: + if keyword in mygroups: + kmask = None + break if kmask: fallback = None @@ -7548,6 +7553,66 @@ def portageexit(): atexit_register(portageexit) +def _gen_missing_encodings(missing_encodings): + + encodings = {} + + if 'ascii' in missing_encodings: + + class AsciiIncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codecs.ascii_encode(input, self.errors)[0] + + class AsciiIncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codecs.ascii_decode(input, self.errors)[0] + + class AsciiStreamWriter(codecs.StreamWriter): + encode = codecs.ascii_encode + + class AsciiStreamReader(codecs.StreamReader): + decode = codecs.ascii_decode + + encodings['ascii'] = codecs.CodecInfo( + name='ascii', + encode=codecs.ascii_encode, + decode=codecs.ascii_decode, + incrementalencoder=AsciiIncrementalEncoder, + incrementaldecoder=AsciiIncrementalDecoder, + streamwriter=AsciiStreamWriter, + streamreader=AsciiStreamReader, + ) + + if 'utf_8' in missing_encodings: + + def utf8decode(input, errors='strict'): + return codecs.utf_8_decode(input, errors, True) + + class Utf8IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codecs.utf_8_encode(input, self.errors)[0] + + class Utf8IncrementalDecoder(codecs.BufferedIncrementalDecoder): + _buffer_decode = codecs.utf_8_decode + + class Utf8StreamWriter(codecs.StreamWriter): + encode = codecs.utf_8_encode + + class Utf8StreamReader(codecs.StreamReader): + decode = codecs.utf_8_decode + + encodings['utf_8'] = codecs.CodecInfo( + name='utf-8', + encode=codecs.utf_8_encode, + decode=utf8decode, + incrementalencoder=Utf8IncrementalEncoder, + incrementaldecoder=Utf8IncrementalDecoder, + streamreader=Utf8StreamWriter, + streamwriter=Utf8StreamReader, + ) + + return encodings + def _ensure_default_encoding(): """ The python that's inside stage 1 or 2 is built with a minimal @@ -7557,53 +7622,56 @@ def _ensure_default_encoding(): LookupError: no codec search functions registered: can't find encoding In order to solve this problem, detect it early and manually register - a search function for the ascii codec. Starting with python-3.0 this - problem is more noticeable because of stricter handling of encoding + a search function for the ascii and utf_8 codecs. Starting with python-3.0 + this problem is more noticeable because of stricter handling of encoding and decoding between strings of characters and bytes. """ - import codecs - try: - codecs.lookup(sys.getdefaultencoding()) - except LookupError: - pass - else: - return - - class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codecs.ascii_encode(input, self.errors)[0] + default_fallback = 'utf_8' + default_encoding = sys.getdefaultencoding() + required_encodings = set(['ascii', 'utf_8']) + required_encodings.add(default_encoding) + missing_encodings = set() + for codec_name in required_encodings: + try: + codecs.lookup(codec_name) + except LookupError: + missing_encodings.add(codec_name) - class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.ascii_decode(input, self.errors)[0] + if not missing_encodings: + return - class StreamWriter(codecs.StreamWriter): - encode = codecs.ascii_encode + encodings = _gen_missing_encodings(missing_encodings) - class StreamReader(codecs.StreamReader): - decode = codecs.ascii_decode + if default_encoding in missing_encodings and \ + default_encoding not in encodings: + # Make the fallback codec correspond to whatever name happens + # to be returned by sys.getdefaultencoding(). - # The sys.setdefaultencoding() function doesn't necessarily exist, - # so just setup the ascii codec to correspond to whatever name - # happens to be returned by sys.getdefaultencoding(). - encoding = sys.getdefaultencoding() + try: + encodings[default_encoding] = codecs.lookup(default_fallback) + except LookupError: + encodings[default_encoding] = encodings[default_fallback] def search_function(name): - if name != encoding: - return None - return codecs.CodecInfo( - name=encoding, - encode=codecs.ascii_encode, - decode=codecs.ascii_decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamwriter=StreamWriter, - streamreader=StreamReader, - ) + codec_info = encodings.get(name) + if codec_info is not None: + return codecs.CodecInfo( + name=codec_info.name, + encode=codec_info.encode, + decode=codec_info.decode, + incrementalencoder=codec_info.incrementalencoder, + incrementaldecoder=codec_info.incrementaldecoder, + streamreader=codec_info.streamreader, + streamwriter=codec_info.streamwriter, + ) + return None codecs.register(search_function) + del codec_name, default_encoding, default_fallback, missing_encodings, \ + required_encodings, search_function + def _global_updates(trees, prev_mtimes): """ Perform new global updates if they exist in $PORTDIR/profiles/updates/. diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index af7b1af47..6de100d96 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1318,7 +1318,7 @@ class vardbapi(dbapi): counter, = self.aux_get(cpv, aux_keys) except KeyError: continue - h.update(counter.encode()) + h.update(counter) return h.hexdigest() def cpv_inject(self, mycpv): @@ -1916,7 +1916,9 @@ class vardbapi(dbapi): def _hash_str(self, s): h = self._new_hash() - h.update(s.encode()) + # Always use a constant utf_8 encoding here, since + # the "default" encoding can change. + h.update(s) h = h.hexdigest() h = h[-self._hex_chars:] h = int(h, 16) -- 2.26.2