From 57ccf7df5c30359f18906fec8ce4870ff92af8c7 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 14 Aug 2009 20:56:01 +0000 Subject: [PATCH] Merged from trunk -r14018:14028 | 14019 | Create portage._merge_encoding and a corresponding _os_merge | | zmedico | wrapper that will later be used for all the merge code. | | 14020 | Use portage's selinux wrapper module for mkdir calls. | | zmedico | | | 14021 | Make _ensure_default_encoding() provide a fallback for the | | zmedico | codec returned by sys.getfilesystemencoding(). | | 14022 | Replace the selinux.spawn() function with a spawn_wrapper() | | zmedico | function and use it inside portage._spawn_fetch() and | | | portage.spawn(). | | 14023 | Don't use the unicode-wrapped os and shutil modules here | | zmedico | since the whole _selinux module itself will be wrapped. | | 14024 | Add an 'encoding' parameter to movefile(), so that we can | | zmedico | make it obey sys.getfilesystemencoding() when called for | | | merge/unmerge. | | 14025 | Make sure portage.selinux and _selinux exist, even when | | zmedico | import fails, so that 'from portage import selinux' never | | | fails inside vartree.py. | | 14026 | Don't 'del sys.modules["selinux"]' when selinux support | | zmedico | isn't enabled in the portage config, because that just | | | assumes too much. | | 14027 | Use portage._merge_encoding to encode/decode all filenames | | zmedico | during merge/unmerge operations. | | 14028 | _merge_encoding = sys.getfilesystemencoding() | | zmedico | | svn path=/main/branches/prefix/; revision=14046 --- pym/portage/__init__.py | 109 +++++++++++++++++++++-------------- pym/portage/_selinux.py | 27 +++++---- pym/portage/checksum.py | 4 ++ pym/portage/dbapi/vartree.py | 77 +++++++++++++++++++------ pym/portage/util.py | 12 +++- 5 files changed, 159 insertions(+), 70 deletions(-) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 0c068b698..47b635941 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -197,21 +197,29 @@ class _unicode_module_wrapper(object): result = _unicode_func_wrapper(result, encoding=encoding) return result -if sys.hexversion >= 0x3000000: - def _unicode_func_wrapper(func): - return func - def _unicode_module_wrapper(mod): - return mod +_merge_encoding = sys.getfilesystemencoding() -import os -os = _unicode_module_wrapper(os, overrides={id(os.read):os.read}) -import shutil -shutil = _unicode_module_wrapper(shutil) +import os as _os +_os_overrides = { + id(_os.fdopen) : _os.fdopen, + id(_os.read) : _os.read, + id(_os.system) : _os.system, +} -# Imports below this point rely on the above unicode wrapper definitions. +os = _unicode_module_wrapper(_os, overrides=_os_overrides) +_os_merge = _unicode_module_wrapper(_os, + encoding=_merge_encoding, overrides=_os_overrides) +import shutil as _shutil +shutil = _unicode_module_wrapper(_shutil) + +# Imports below this point rely on the above unicode wrapper definitions. +_selinux = None +selinux = None +_selinux_merge = _unicode_module_wrapper(_selinux, encoding=_merge_encoding) try: - import portage._selinux as selinux + import portage._selinux + selinux = _unicode_module_wrapper(_selinux) except OSError, e: sys.stderr.write("!!! SELinux not loaded: %s\n" % str(e)) del e @@ -308,8 +316,10 @@ def _ensure_default_encoding(): default_fallback = 'utf_8' default_encoding = sys.getdefaultencoding().lower().replace('-', '_') + filesystem_encoding = sys.getfilesystemencoding().lower().replace('-', '_') required_encodings = set(['ascii', 'utf_8']) required_encodings.add(default_encoding) + required_encodings.add(filesystem_encoding) missing_encodings = set() for codec_name in required_encodings: try: @@ -325,13 +335,23 @@ def _ensure_default_encoding(): if default_encoding in missing_encodings and \ default_encoding not in encodings: # Make the fallback codec correspond to whatever name happens - # to be returned by sys.getdefaultencoding(). + # to be returned by sys.getfilesystemencoding(). try: encodings[default_encoding] = codecs.lookup(default_fallback) except LookupError: encodings[default_encoding] = encodings[default_fallback] + if filesystem_encoding in missing_encodings and \ + filesystem_encoding not in encodings: + # Make the fallback codec correspond to whatever name happens + # to be returned by sys.getdefaultencoding(). + + try: + encodings[filesystem_encoding] = codecs.lookup(default_fallback) + except LookupError: + encodings[filesystem_encoding] = encodings[default_fallback] + def search_function(name): name = name.lower() name = name.replace('-', '_') @@ -350,7 +370,8 @@ def _ensure_default_encoding(): codecs.register(search_function) - del codec_name, default_encoding, default_fallback, missing_encodings, \ + del codec_name, default_encoding, default_fallback, \ + filesystem_encoding, missing_encodings, \ required_encodings, search_function # Do this ASAP since writemsg() might not work without it. @@ -381,9 +402,9 @@ if platform.system() in ('FreeBSD',) and rootuid == 0: return # Try to generate an ENOENT error if appropriate. if 'h' in opts: - os.lstat(path) + _os_merge.lstat(path) else: - os.stat(path) + _os_merge.stat(path) # Make sure the binary exists. if not portage.process.find_binary('chflags'): raise portage.exception.CommandNotFound('chflags') @@ -3666,7 +3687,7 @@ class config(object): if getattr(self, "_selinux_enabled", None) is None: self._selinux_enabled = 0 if "selinux" in self["USE"].split(): - if "selinux" in globals(): + if selinux: if selinux.is_selinux_enabled() == 1: self._selinux_enabled = 1 else: @@ -3675,11 +3696,7 @@ class config(object): writemsg("!!! SELinux module not found. Please verify that it was installed.\n", noiselevel=-1) self._selinux_enabled = 0 - if self._selinux_enabled == 0: - try: - del sys.modules["selinux"] - except KeyError: - pass + return self._selinux_enabled if sys.hexversion >= 0x3000000: @@ -3873,8 +3890,8 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero spawn_func = portage.process.spawn_sandbox if sesandbox: - con = selinux.settype(mysettings["PORTAGE_SANDBOX_T"]) - selinux.setexec(con) + spawn_func = selinux.spawn_wrapper(spawn_func, + mysettings["PORTAGE_SANDBOX_T"]) returnpid = keywords.get("returnpid") keywords["returnpid"] = True @@ -3883,8 +3900,6 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero finally: if logfile: os.close(slave_fd) - if sesandbox: - selinux.setexec() if returnpid: return mypids @@ -3959,21 +3974,17 @@ def _spawn_fetch(settings, args, **kwargs): os.getuid() == 0 and portage_gid and portage_uid: kwargs.update(_userpriv_spawn_kwargs) - try: + spawn_func = portage.process.spawn - if settings.selinux_enabled(): - con = selinux.settype(settings["PORTAGE_FETCH_T"]) - selinux.setexec(con) - # bash is an allowed entrypoint, while most binaries are not - if args[0] != BASH_BINARY: - args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args + if settings.selinux_enabled(): + spawn_func = selinux.spawn_wrapper(spawn_func, + settings["PORTAGE_FETCH_T"]) - rval = portage.process.spawn(args, - env=dict(settings.iteritems()), **kwargs) + # bash is an allowed entrypoint, while most binaries are not + if args[0] != BASH_BINARY: + args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args - finally: - if settings.selinux_enabled(): - selinux.setexec() + rval = spawn_func(args, env=dict(settings.iteritems()), **kwargs) return rval @@ -5544,6 +5555,9 @@ def _post_src_install_uid_fix(mysettings): S_ISUID and S_ISGID bits, so those bits are restored if necessary. """ + + os = _os_merge + inst_uid = int(mysettings["PORTAGE_INST_UID"]) inst_gid = int(mysettings["PORTAGE_INST_GID"]) @@ -5563,9 +5577,9 @@ def _post_src_install_uid_fix(mysettings): counted_inodes = set() for parent, dirs, files in os.walk(destdir): - parent = _unicode_decode(parent) + parent = _unicode_decode(parent, encoding=_merge_encoding) for fname in chain(dirs, files): - fname = _unicode_decode(fname) + fname = _unicode_decode(fname, encoding=_merge_encoding) fpath = os.path.join(parent, fname) mystat = os.lstat(fpath) if stat.S_ISREG(mystat.st_mode) and \ @@ -5581,7 +5595,9 @@ def _post_src_install_uid_fix(mysettings): myuid = inst_uid if mystat.st_gid == portage_gid: mygid = inst_gid - apply_secpass_permissions(fpath, uid=myuid, gid=mygid, + apply_secpass_permissions( + _unicode_encode(fpath, encoding=_merge_encoding), + uid=myuid, gid=mygid, mode=mystat.st_mode, stat_cached=mystat, follow_links=False) @@ -7041,16 +7057,25 @@ def _movefile(src, dest, **kwargs): "mv '%s' '%s'" % (src, dest)) def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, - hardlink_candidates=None): + hardlink_candidates=None, encoding='utf_8'): """moves a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns true on success and false on failure. Move is atomic.""" #print "movefile("+str(src)+","+str(dest)+","+str(newmtime)+","+str(sstat)+")" - global lchown + if mysettings is None: global settings mysettings = settings + selinux_enabled = mysettings.selinux_enabled() + if selinux_enabled: + selinux = _unicode_module_wrapper(_selinux, encoding=encoding) + + lchown = _unicode_func_wrapper(data.lchown, encoding=encoding) + os = _unicode_module_wrapper(_os, + encoding=encoding, overrides=_os_overrides) + shutil = _unicode_module_wrapper(_shutil, encoding=encoding) + try: if not sstat: sstat=os.lstat(src) diff --git a/pym/portage/_selinux.py b/pym/portage/_selinux.py index e3c35ec5d..e92c21fc9 100644 --- a/pym/portage/_selinux.py +++ b/pym/portage/_selinux.py @@ -2,9 +2,12 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +# Don't use the unicode-wrapped os and shutil modules here since +# the whole _selinux module itself will be wrapped. +import os +import shutil + import portage -from portage import os -from portage import shutil import selinux from selinux import is_selinux_enabled, getfilecon, lgetfilecon @@ -73,14 +76,18 @@ def setfscreate(ctx="\n"): raise OSError( "setfscreate: Failed setting fs create context \"%s\"." % ctx) -def spawn(selinux_type, spawn_func, mycommand, opt_name=None, **keywords): - selinux_type = portage._unicode_encode(selinux_type) - con = settype(selinux_type) - setexec(con) - try: - return spawn_func(mycommand, opt_name=opt_name, **keywords) - finally: - setexec() +def spawn_wrapper(spawn_func, selinux_type): + + def wrapper_func(*args, **kwargs): + selinux_type = portage._unicode_encode(selinux_type) + con = settype(selinux_type) + setexec(con) + try: + return spawn_func(*args, **kwargs) + finally: + setexec() + + return wrapper_func def symlink(target, link, reflnk): target = portage._unicode_encode(target) diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 501ae3e6a..5def1ac8a 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -6,6 +6,7 @@ import portage from portage.const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE from portage import os +from portage import _merge_encoding from portage import _unicode_encode import errno import stat @@ -117,6 +118,9 @@ if os.path.exists(PRELINK_BINARY): def perform_md5(x, calc_prelink=0): return perform_checksum(x, "MD5", calc_prelink)[0] +def _perform_md5_merge(x, **kwargs): + return perform_md5(_unicode_encode(x, encoding=_merge_encoding), **kwargs) + def perform_all(x, calc_prelink=0): mydict = {} for k in hashfunc_map: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index b2419af69..220745aba 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -9,7 +9,7 @@ __all__ = ["PreservedLibsRegistry", "LinkageMap", import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:perform_md5', + 'portage.checksum:_perform_md5_merge@perform_md5', 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ 'use_reduce,paren_reduce', 'portage.elog:elog_process', @@ -38,6 +38,9 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ # This is a special version of the os module, wrapped for unicode support. from portage import os +from portage import _merge_encoding +from portage import _os_merge +from portage import _selinux_merge from portage import _unicode_encode from portage.cache.mappings import slot_dict_class @@ -140,6 +143,9 @@ class PreservedLibsRegistry(object): def pruneNonExisting(self): """ Remove all records for objects that no longer exist on the filesystem. """ + + os = _os_merge + for cps in self._data.keys(): cpv, counter, paths = self._data[cps] paths = [f for f in paths \ @@ -239,6 +245,9 @@ class LinkageMap(object): 2. realpath of object if object does not exist. """ + + os = _os_merge + abs_path = os.path.join(root, obj.lstrip(os.sep)) try: object_stat = os.stat(abs_path) @@ -276,6 +285,8 @@ class LinkageMap(object): Raises CommandNotFound if there are preserved libs and the scanelf binary is not available. """ + + os = _os_merge root = self._root root_len = len(root) - 1 self._clear_cache() @@ -395,6 +406,9 @@ class LinkageMap(object): object that have no corresponding libraries to fulfill the dependency. """ + + os = _os_merge + class _LibraryCache(object): """ @@ -541,6 +555,7 @@ class LinkageMap(object): 2. False if obj is not a master link """ + os = _os_merge basename = os.path.basename(obj) obj_key = self._obj_key(obj) if obj_key not in self._obj_properties: @@ -608,6 +623,9 @@ class LinkageMap(object): set-of-library-paths satisfy soname. """ + + os = _os_merge + rValue = {} if not self._libs: @@ -668,6 +686,9 @@ class LinkageMap(object): set-of-library-paths satisfy soname. """ + + os = _os_merge + rValue = set() if not self._libs: @@ -2962,6 +2983,7 @@ class dblink(object): @rtype: None """ + os = _os_merge showMessage = self._display_merge scheduler = self._scheduler @@ -3240,6 +3262,8 @@ class dblink(object): if the file is not owned by this package. """ + os = _os_merge + filename = portage._unicode_decode(filename) destroot = portage._unicode_decode(destroot) @@ -3324,6 +3348,7 @@ class dblink(object): "preserve-libs" in self.settings.features): return None + os = _os_merge linkmap = self.vartree.dbapi.linkmap installed_instance = self._installed_instance old_contents = installed_instance.getcontents() @@ -3425,6 +3450,7 @@ class dblink(object): if not preserve_paths: return + os = _os_merge showMessage = self._display_merge root = self.myroot @@ -3432,7 +3458,7 @@ class dblink(object): new_contents = self.getcontents().copy() old_contents = self._installed_instance.getcontents() for f in sorted(preserve_paths): - f = portage._unicode_decode(f) + f = portage._unicode_decode(f, encoding=_merge_encoding) f_abs = os.path.join(root, f.lstrip(os.sep)) contents_entry = old_contents.get(f_abs) if contents_entry is None: @@ -3577,6 +3603,8 @@ class dblink(object): Remove files returned from _find_unused_preserved_libs(). """ + os = _os_merge + files_to_remove = set() for files in cpv_lib_map.itervalues(): files_to_remove.update(files) @@ -3618,6 +3646,9 @@ class dblink(object): self.vartree.dbapi.plib_registry.pruneNonExisting() def _collision_protect(self, srcroot, destroot, mypkglist, mycontents): + + os = _os_merge + collision_ignore = set([normalize_path(myignore) for myignore in \ portage.util.shlex_split( self.settings.get("COLLISION_IGNORE", ""))]) @@ -3729,6 +3760,9 @@ class dblink(object): Multiple paths may reference the same inode due to hardlinks. All lstat() calls are relative to self.myroot. """ + + os = _os_merge + root = self.myroot inode_map = {} for f in path_iter: @@ -3752,6 +3786,8 @@ class dblink(object): if not installed_instances: return 0 + os = _os_merge + showMessage = self._display_merge scheduler = self._scheduler @@ -3862,6 +3898,8 @@ class dblink(object): not existing; we will merge these symlinks at a later time. """ + os = _os_merge + srcroot = portage._unicode_decode(srcroot) destroot = portage._unicode_decode(destroot) inforoot = portage._unicode_decode(inforoot) @@ -3960,9 +3998,9 @@ class dblink(object): def onerror(e): raise for parent, dirs, files in os.walk(srcroot, onerror=onerror): - parent = portage._unicode_decode(parent) + parent = portage._unicode_decode(parent, encoding=_merge_encoding) for f in files: - f = portage._unicode_decode(f) + f = portage._unicode_decode(f, encoding=_merge_encoding) file_path = os.path.join(parent, f) relative_path = file_path[srcroot_len:] @@ -4442,8 +4480,9 @@ class dblink(object): writemsg = self._display_merge scheduler = self._scheduler - sep = portage.os.sep - join = portage.os.path.join + os = _os_merge + sep = os.sep + join = os.path.join srcroot = normalize_path(srcroot).rstrip(sep) + sep destroot = normalize_path(destroot).rstrip(sep) + sep @@ -4526,7 +4565,9 @@ class dblink(object): secondhand.append(mysrc[len(srcroot):]) continue # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly - mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) + mymtime = movefile(mysrc, mydest, newmtime=thismtime, + sstat=mystat, mysettings=self.settings, + encoding=_merge_encoding) if mymtime != None: showMessage(">>> %s -> %s\n" % (mydest, myto)) outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") @@ -4564,15 +4605,15 @@ class dblink(object): bsd_chflags.lchflags(mydest, dflags) else: # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. - if movefile(mydest, mydest+".backup", mysettings=self.settings) is None: + if movefile(mydest, mydest+".backup", + mysettings=self.settings, + encoding=_merge_encoding) is None: return 1 showMessage(_("bak %s %s.backup\n") % (mydest, mydest), level=logging.ERROR, noiselevel=-1) #now create our directory if self.settings.selinux_enabled(): - import selinux - sid = selinux.get_sid(mysrc) - selinux.secure_mkdir(mydest,sid) + _selinux_merge.mkdir(mydest, mysrc) else: os.mkdir(mydest) if bsd_chflags: @@ -4583,9 +4624,7 @@ class dblink(object): else: #destination doesn't exist if self.settings.selinux_enabled(): - import selinux - sid = selinux.get_sid(mysrc) - selinux.secure_mkdir(mydest, sid) + _selinux_merge.mkdir(mydest, mysrc) else: os.mkdir(mydest) os.chmod(mydest, mystat[0]) @@ -4672,7 +4711,8 @@ class dblink(object): mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, - hardlink_candidates=hardlink_candidates) + hardlink_candidates=hardlink_candidates, + encoding=_merge_encoding) if mymtime is None: return 1 if hardlink_candidates is not None: @@ -4687,7 +4727,9 @@ class dblink(object): zing = "!!!" if mydmode is None: # destination doesn't exist - if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) != None: + if movefile(mysrc, mydest, newmtime=thismtime, + sstat=mystat, mysettings=self.settings, + encoding=_merge_encoding) is not None: zing = ">>>" else: return 1 @@ -4838,6 +4880,7 @@ def write_contents(contents, root, f): f.write(line) def tar_contents(contents, root, tar, protect=None, onProgress=None): + os = _os_merge from portage.util import normalize_path import tarfile root = normalize_path(root).rstrip(os.path.sep) + os.path.sep @@ -4884,7 +4927,7 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tarinfo.size = 0 tar.addfile(tarinfo) else: - f = open(_unicode_encode(path), 'rb') + f = open(_unicode_encode(path, encoding=_merge_encoding), 'rb') try: tar.addfile(tarinfo, f) finally: diff --git a/pym/portage/util.py b/pym/portage/util.py index 15d67fdfb..d99e8bbb5 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -24,6 +24,8 @@ import sys import portage from portage import os +from portage import _merge_encoding +from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode from portage.exception import PortageException, FileNotFound, \ @@ -1204,6 +1206,9 @@ class ConfigProtect(object): def updateprotect(self): """Update internal state for isprotected() calls. Nonexistent paths are ignored.""" + + os = _os_merge + self.protect = [] self._dirs = set() for x in self.protect_list: @@ -1280,6 +1285,9 @@ def new_protect_filename(mydest, newmd5=None): # config protection filename format: # ._cfg0000_foo # 0123456789012 + + os = _os_merge + prot_num = -1 last_pfile = "" @@ -1308,7 +1316,7 @@ def new_protect_filename(mydest, newmd5=None): if last_pfile and newmd5: import portage.checksum try: - last_pfile_md5 = portage.checksum.perform_md5(old_pfile) + last_pfile_md5 = portage.checksum._perform_md5_merge(old_pfile) except FileNotFound: # The file suddenly disappeared or it's a broken symlink. pass @@ -1327,6 +1335,8 @@ def find_updated_config_files(target_root, config_protect): If no configuration files needs to be updated, None is returned """ + os = _os_merge + if config_protect: # directories with some protect files in them for x in config_protect: -- 2.26.2