From 0915a2a5ca0f2ec30160bcace8d5f3d26616af05 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 24 Aug 2009 09:50:34 +0000 Subject: [PATCH] Merged from trunk -r14131:14140 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit | 14132 | Add a LIVEVCS.unmasked category for ebuilds that have | | zmedico | non-empty KEYWORDS and are not masked in the global | | | package.mask. Thanks to Diego E. Pettenò for the | | | suggestion. | | 14133 | Bug #282306 - Inside dblink._match_contents(), fall back to | | zmedico | utf_8 encoding if a path cannot be encoded under the user's | | | chosen encoding. This should fix the traceback shown in bug | | | #281199, comment #26. | | 14134 | Bug #282505 - Fix unicode handling for package moves in | | zmedico | binary packages. | | 14140 | remove FIXME, since zmedico fixed it | | grobian | | svn path=/main/branches/prefix/; revision=14148 --- bin/repoman | 27 ++++++++++++- man/repoman.1 | 4 ++ pym/portage/dbapi/bintree.py | 18 ++++++--- pym/portage/dbapi/vartree.py | 78 ++++++++++++++++++++++++++++++++---- pym/portage/output.py | 5 --- pym/portage/update.py | 12 ++++-- pym/portage/xpak.py | 11 +++++ 7 files changed, 134 insertions(+), 21 deletions(-) diff --git a/bin/repoman b/bin/repoman index 7f1ba3b8f..410dcf8a1 100755 --- a/bin/repoman +++ b/bin/repoman @@ -300,6 +300,7 @@ qahelp={ "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set", "variable.readonly":"Assigning a readonly variable", "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.", + "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.", "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file", "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)", "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.", @@ -362,7 +363,8 @@ qawarnings = set(( "virtual.unavailable", "usage.obsolete", "upstream.workaround", -"LIVEVCS.stable" +"LIVEVCS.stable", +"LIVEVCS.unmasked", )) non_ascii_re = re.compile(r'[^\x00-\x7f]') @@ -570,6 +572,7 @@ valid_profile_types = frozenset(['dev', 'exp', 'stable']) kwlist = set() liclist = set() uselist = set() +global_pmasklines = [] for path in portdb.porttrees: try: @@ -600,6 +603,9 @@ for path in portdb.porttrees: if x: uselist.add(use_prefix + x[0]) + global_pmasklines.append(portage.util.grabfile_package( + os.path.join(path, 'profiles', 'package.mask'), recursive=1)) + desc_path = os.path.join(path, 'profiles', 'profiles.desc') try: desc_file = codecs.open(_unicode_encode(desc_path, @@ -635,6 +641,21 @@ for path in portdb.porttrees: repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist)) repoman_settings.backup_changes('PORTAGE_ARCHLIST') +global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1) +global_pmaskdict = {} +for x in global_pmasklines: + global_pmaskdict.setdefault(portage.dep_getkey(x), []).append(x) +del global_pmasklines + +def has_global_mask(pkg): + mask_atoms = global_pmaskdict.get(pkg.cp) + if mask_atoms: + pkg_list = [pkg] + for x in mask_atoms: + if portage.dep.match_from_list(x, pkg_list): + return x + return None + # Ensure that profile sub_path attributes are unique. Process in reverse order # so that profiles with duplicate sub_path from overlays will override # profiles with the same sub_path from parent repos. @@ -1347,6 +1368,10 @@ for x in scanlist: bad_stable_keywords) del bad_stable_keywords + if keywords and not has_global_mask(pkg): + stats["LIVEVCS.unmasked"] += 1 + fails["LIVEVCS.unmasked"].append(relative_path) + if options.ignore_arches: arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"], repoman_settings["ACCEPT_KEYWORDS"].split()]] diff --git a/man/repoman.1 b/man/repoman.1 index 13383a423..52ef02a1e 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -149,6 +149,10 @@ Syntax error in LICENSE (usually an extra/missing space/parenthesis) .B LIVEVCS.stable Ebuild is a live ebuild (cvs, git, darcs, svn, etc) checkout with stable keywords. .TP +.B LIVEVCS.unmasked +Ebuild is a live ebuild (cvs, git, darcs, svn, etc) checkout but has keywords +and is not masked in the global package.mask. +.TP .B PDEPEND.bad User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) .TP diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 8de25b7f6..aeb088e83 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -266,13 +266,21 @@ class binarytree(object): mydata = mytbz2.get_data() updated_items = update_dbentries([mylist], mydata) mydata.update(updated_items) - mydata["PF"] = mynewpkg + "\n" - mydata["CATEGORY"] = mynewcat+"\n" + mydata[_unicode_encode('PF', + encoding=_encodings['repo.content'])] = \ + _unicode_encode(mynewpkg + "\n", + encoding=_encodings['repo.content']) + mydata[_unicode_encode('CATEGORY', + encoding=_encodings['repo.content'])] = \ + _unicode_encode(mynewcat + "\n", + encoding=_encodings['repo.content']) if mynewpkg != myoldpkg: - ebuild_data = mydata.get(myoldpkg+".ebuild") + ebuild_data = mydata.pop(_unicode_encode(myoldpkg + '.ebuild', + encoding=_encodings['repo.content']), None) if ebuild_data is not None: - mydata[mynewpkg+".ebuild"] = ebuild_data - del mydata[myoldpkg+".ebuild"] + mydata[_unicode_encode(mynewpkg + '.ebuild', + encoding=_encodings['repo.content'])] = ebuild_data + mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) self.dbapi.cpv_remove(mycpv) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 47c48a99f..04fccd128 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -3339,23 +3339,65 @@ class dblink(object): if the file is not owned by this package. """ - os = _os_merge - filename = _unicode_decode(filename, encoding=_encodings['content'], errors='strict') destroot = _unicode_decode(destroot, encoding=_encodings['content'], errors='strict') + # The given filename argument might have a different encoding than the + # the filenames contained in the contents, so use separate wrapped os + # modules for each. The basename is more likely to contain non-ascii + # characters than the directory path, so use os_filename_arg for all + # operations involving the basename of the filename arg. + os_filename_arg = _os_merge + os = _os_merge + + try: + _unicode_encode(filename, + encoding=_encodings['merge'], errors='strict') + except UnicodeEncodeError: + # The package appears to have been merged with a + # different value of sys.getfilesystemencoding(), + # so fall back to utf_8 if appropriate. + try: + _unicode_encode(filename, + encoding=_encodings['fs'], errors='strict') + except UnicodeEncodeError: + pass + else: + os_filename_arg = portage.os + destfile = normalize_path( - os.path.join(destroot, filename.lstrip(os.path.sep))) + os_filename_arg.path.join(destroot, + filename.lstrip(os_filename_arg.path.sep))) pkgfiles = self.getcontents() if pkgfiles and destfile in pkgfiles: return destfile if pkgfiles: - basename = os.path.basename(destfile) + basename = os_filename_arg.path.basename(destfile) if self._contents_basenames is None: + + try: + for x in pkgfiles: + _unicode_encode(x, + encoding=_encodings['merge'], + errors='strict') + except UnicodeEncodeError: + # The package appears to have been merged with a + # different value of sys.getfilesystemencoding(), + # so fall back to utf_8 if appropriate. + try: + for x in pkgfiles: + _unicode_encode(x, + encoding=_encodings['fs'], + errors='strict') + except UnicodeEncodeError: + pass + else: + os = portage.os + self._contents_basenames = set( os.path.basename(x) for x in pkgfiles) if basename not in self._contents_basenames: @@ -3366,15 +3408,36 @@ class dblink(object): # Use stat rather than lstat since we want to follow # any symlinks to the real parent directory. - parent_path = os.path.dirname(destfile) + parent_path = os_filename_arg.path.dirname(destfile) try: - parent_stat = os.stat(parent_path) + parent_stat = os_filename_arg.stat(parent_path) except EnvironmentError, e: if e.errno != errno.ENOENT: raise del e return False if self._contents_inodes is None: + + if os is _os_merge: + try: + for x in pkgfiles: + _unicode_encode(x, + encoding=_encodings['merge'], + errors='strict') + except UnicodeEncodeError: + # The package appears to have been merged with a + # different value of sys.getfilesystemencoding(), + # so fall back to utf_8 if appropriate. + try: + for x in pkgfiles: + _unicode_encode(x, + encoding=_encodings['fs'], + errors='strict') + except UnicodeEncodeError: + pass + else: + os = portage.os + self._contents_inodes = {} parent_paths = set() for x in pkgfiles: @@ -3396,11 +3459,12 @@ class dblink(object): self._contents_inodes[inode_key] = p_path_list if p_path not in p_path_list: p_path_list.append(p_path) + p_path_list = self._contents_inodes.get( (parent_stat.st_dev, parent_stat.st_ino)) if p_path_list: for p_path in p_path_list: - x = os.path.join(p_path, basename) + x = os_filename_arg.path.join(p_path, basename) if x in pkgfiles: return x diff --git a/pym/portage/output.py b/pym/portage/output.py index 224e44380..d18cbc0d5 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -152,11 +152,6 @@ def _parse_color_map(config_root='/', onerror=None): @rtype: dict @return: a dictionary mapping color classes to color codes """ - # FIXME: use config_root iso / (breaks Prefix) - # We can use ObjectProxy to implement lazy initialization of - # codes and _styles, and add an init(config_root="/") function - # that can be called in order adjust the location that color.map - # is read from. global codes, _styles myfile = os.path.join(config_root, COLOR_MAP_FILE) ansi_code_pattern = re.compile("^[0-9;]*m$") diff --git a/pym/portage/update.py b/pym/portage/update.py index ca67370cd..56e6709c0 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -55,12 +55,18 @@ def update_dbentries(update_iter, mydata): dict containing only the updated items.""" updated_items = {} for k, mycontent in mydata.iteritems(): - if k not in ignored_dbentries: + k_unicode = _unicode_decode(k, + encoding=_encodings['repo.content'], errors='replace') + if k_unicode not in ignored_dbentries: + mycontent = _unicode_decode(mycontent, + encoding=_encodings['repo.content'], errors='replace') orig_content = mycontent for update_cmd in update_iter: mycontent = update_dbentry(update_cmd, mycontent) if mycontent != orig_content: - updated_items[k] = mycontent + updated_items[k] = _unicode_encode(mycontent, + encoding=_encodings['repo.content'], + errors='backslashreplace') return updated_items def fixdbentries(update_iter, dbdir): @@ -77,7 +83,7 @@ def fixdbentries(update_iter, dbdir): updated_items = update_dbentries(update_iter, mydata) for myfile, mycontent in updated_items.iteritems(): file_path = os.path.join(dbdir, myfile) - write_atomic(file_path, mycontent) + write_atomic(file_path, mycontent, encoding=_encodings['repo.content']) return len(updated_items) > 0 def grab_updates(updpath, prev_mtimes=None): diff --git a/pym/portage/xpak.py b/pym/portage/xpak.py index 5b08c0a3f..15a432f51 100644 --- a/pym/portage/xpak.py +++ b/pym/portage/xpak.py @@ -97,6 +97,17 @@ def xpak(rootdir,outfile=None): def xpak_mem(mydata): """Create an xpack segement from a map object.""" + + mydata_encoded = {} + for k, v in mydata.iteritems(): + k = _unicode_encode(k, + encoding=_encodings['repo.content'], errors='backslashreplace') + v = _unicode_encode(v, + encoding=_encodings['repo.content'], errors='backslashreplace') + mydata_encoded[k] = v + mydata = mydata_encoded + del mydata_encoded + indexglob = _unicode_encode('') indexpos=0 dataglob = _unicode_encode('') -- 2.26.2