pym/portage/util/movefile.py

   1 # Copyright 2010-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 __all__ = ['movefile']
   7
   8 import errno
   9 import fnmatch
  10 import os as _os
  11 import shutil as _shutil
  12 import stat
  13 import sys
  14 import subprocess
  15 import textwrap
  16
  17 import portage
  18 from portage import bsd_chflags, _encodings, _os_overrides, _selinux, \
  19         _unicode_decode, _unicode_encode, _unicode_func_wrapper,\
  20         _unicode_module_wrapper
  21 from portage.const import MOVE_BINARY
  22 from portage.exception import OperationNotSupported
  23 from portage.localization import _
  24 from portage.process import spawn
  25 from portage.util import writemsg
  26
  27 def _apply_stat(src_stat, dest):
  28         _os.chown(dest, src_stat.st_uid, src_stat.st_gid)
  29         _os.chmod(dest, stat.S_IMODE(src_stat.st_mode))
  30
  31 _xattr_excluder_cache = {}
  32
  33 def _get_xattr_excluder(pattern):
  34
  35         try:
  36                 value = _xattr_excluder_cache[pattern]
  37         except KeyError:
  38                 value = _xattr_excluder(pattern)
  39                 _xattr_excluder_cache[pattern] = value
  40
  41         return value
  42
  43 class _xattr_excluder(object):
  44
  45         __slots__ = ('_pattern_split',)
  46
  47         def __init__(self, pattern):
  48
  49                 if pattern is None:
  50                         self._pattern_split = None
  51                 else:
  52                         pattern = pattern.split()
  53                         if not pattern:
  54                                 self._pattern_split = None
  55                         else:
  56                                 pattern.sort()
  57                                 self._pattern_split = tuple(pattern)
  58
  59         def __call__(self, attr):
  60
  61                 if self._pattern_split is None:
  62                         return False
  63
  64                 match = fnmatch.fnmatch
  65                 for x in self._pattern_split:
  66                         if match(attr, x):
  67                                 return True
  68
  69                 return False
  70
  71 if hasattr(_os, "getxattr"):
  72         # Python >=3.3 and GNU/Linux
  73         def _copyxattr(src, dest, exclude=None):
  74
  75                 attrs = _os.listxattr(src)
  76                 if attrs:
  77                         if exclude is not None and isinstance(attrs[0], bytes):
  78                                 exclude = exclude.encode(_encodings['fs'])
  79                         exclude = _get_xattr_excluder(exclude)
  80
  81                 for attr in attrs:
  82                         if exclude(attr):
  83                                 continue
  84                         try:
  85                                 _os.setxattr(dest, attr, _os.getxattr(src, attr))
  86                                 raise_exception = False
  87                         except OSError:
  88                                 raise_exception = True
  89                         if raise_exception:
  90                                 raise OperationNotSupported(_("Filesystem containing file '%s' "
  91                                         "does not support extended attribute '%s'") %
  92                                         (_unicode_decode(dest), _unicode_decode(attr)))
  93 else:
  94         try:
  95                 import xattr
  96         except ImportError:
  97                 xattr = None
  98         if xattr is not None:
  99                 def _copyxattr(src, dest, exclude=None):
 100
 101                         try:
 102                                 attrs = xattr.list(src)
 103                         except IOError as e:
 104                                 if e.errno != OperationNotSupported.errno:
 105                                         raise
 106                                 attrs = ()
 107
 108                         if attrs:
 109                                 if exclude is not None and isinstance(attrs[0], bytes):
 110                                         exclude = exclude.encode(_encodings['fs'])
 111                                 exclude = _get_xattr_excluder(exclude)
 112
 113                         for attr in attrs:
 114                                 if exclude(attr):
 115                                         continue
 116                                 try:
 117                                         xattr.set(dest, attr, xattr.get(src, attr))
 118                                         raise_exception = False
 119                                 except IOError:
 120                                         raise_exception = True
 121                                 if raise_exception:
 122                                         raise OperationNotSupported(_("Filesystem containing file '%s' "
 123                                                 "does not support extended attribute '%s'") %
 124                                                 (_unicode_decode(dest), _unicode_decode(attr)))
 125         else:
 126                 _devnull = open("/dev/null", "wb")
 127                 try:
 128                         subprocess.call(["getfattr", "--version"], stdout=_devnull)
 129                         subprocess.call(["setfattr", "--version"], stdout=_devnull)
 130                         _has_getfattr_and_setfattr = True
 131                 except OSError:
 132                         _has_getfattr_and_setfattr = False
 133                 _devnull.close()
 134                 if _has_getfattr_and_setfattr:
 135                         def _copyxattr(src, dest, exclude=None):
 136                                 # TODO: implement exclude
 137                                 getfattr_process = subprocess.Popen(["getfattr", "-d", "--absolute-names", src], stdout=subprocess.PIPE)
 138                                 getfattr_process.wait()
 139                                 extended_attributes = getfattr_process.stdout.readlines()
 140                                 getfattr_process.stdout.close()
 141                                 if extended_attributes:
 142                                         extended_attributes[0] = b"# file: " + _unicode_encode(dest) + b"\n"
 143                                         setfattr_process = subprocess.Popen(["setfattr", "--restore=-"], stdin=subprocess.PIPE, stderr=subprocess.PIPE)
 144                                         setfattr_process.communicate(input=b"".join(extended_attributes))
 145                                         if setfattr_process.returncode != 0:
 146                                                 raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest)
 147                 else:
 148                         def _copyxattr(src, dest, exclude=None):
 149                                 pass
 150
 151 def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
 152                 hardlink_candidates=None, encoding=_encodings['fs']):
 153         """moves a file from src to dest, preserving all permissions and attributes; mtime will
 154         be preserved even when moving across filesystems.  Returns mtime as integer on success
 155         and None on failure.  mtime is expressed in seconds in Python <3.3 and nanoseconds in
 156         Python >=3.3.  Move is atomic."""
 157
 158         if mysettings is None:
 159                 mysettings = portage.settings
 160
 161         src_bytes = _unicode_encode(src, encoding=encoding, errors='strict')
 162         dest_bytes = _unicode_encode(dest, encoding=encoding, errors='strict')
 163         xattr_enabled = "xattr" in mysettings.features
 164         selinux_enabled = mysettings.selinux_enabled()
 165         if selinux_enabled:
 166                 selinux = _unicode_module_wrapper(_selinux, encoding=encoding)
 167                 _copyfile = selinux.copyfile
 168                 _rename = selinux.rename
 169         else:
 170                 _copyfile = _shutil.copyfile
 171                 _rename = _os.rename
 172
 173         lchown = _unicode_func_wrapper(portage.data.lchown, encoding=encoding)
 174         os = _unicode_module_wrapper(_os,
 175                 encoding=encoding, overrides=_os_overrides)
 176
 177         try:
 178                 if not sstat:
 179                         sstat=os.lstat(src)
 180
 181         except SystemExit as e:
 182                 raise
 183         except Exception as e:
 184                 writemsg("!!! %s\n" % _("Stating source file failed... movefile()"),
 185                         noiselevel=-1)
 186                 writemsg("!!! %s\n" % (e,), noiselevel=-1)
 187                 return None
 188
 189         destexists=1
 190         try:
 191                 dstat=os.lstat(dest)
 192         except (OSError, IOError):
 193                 dstat=os.lstat(os.path.dirname(dest))
 194                 destexists=0
 195
 196         if bsd_chflags:
 197                 if destexists and dstat.st_flags != 0:
 198                         bsd_chflags.lchflags(dest, 0)
 199                 # Use normal stat/chflags for the parent since we want to
 200                 # follow any symlinks to the real parent directory.
 201                 pflags = os.stat(os.path.dirname(dest)).st_flags
 202                 if pflags != 0:
 203                         bsd_chflags.chflags(os.path.dirname(dest), 0)
 204
 205         if destexists:
 206                 if stat.S_ISLNK(dstat[stat.ST_MODE]):
 207                         try:
 208                                 os.unlink(dest)
 209                                 destexists=0
 210                         except SystemExit as e:
 211                                 raise
 212                         except Exception as e:
 213                                 pass
 214
 215         if stat.S_ISLNK(sstat[stat.ST_MODE]):
 216                 try:
 217                         target=os.readlink(src)
 218                         if mysettings and "D" in mysettings and \
 219                                 target.startswith(mysettings["D"]):
 220                                 target = target[len(mysettings["D"])-1:]
 221                         if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
 222                                 os.unlink(dest)
 223                         try:
 224                                 if selinux_enabled:
 225                                         selinux.symlink(target, dest, src)
 226                                 else:
 227                                         os.symlink(target, dest)
 228                         except OSError as e:
 229                                 # Some programs will create symlinks automatically, so we have
 230                                 # to tolerate these links being recreated during the merge
 231                                 # process. In any case, if the link is pointing at the right
 232                                 # place, we're in good shape.
 233                                 if e.errno not in (errno.ENOENT, errno.EEXIST) or \
 234                                         target != os.readlink(dest):
 235                                         raise
 236                         lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
 237
 238                         try:
 239                                 _os.unlink(src_bytes)
 240                         except OSError:
 241                                 pass
 242
 243                         if sys.hexversion >= 0x3030000:
 244                                 try:
 245                                         os.utime(dest, ns=(sstat.st_mtime_ns, sstat.st_mtime_ns), follow_symlinks=False)
 246                                 except NotImplementedError:
 247                                         # utimensat() and lutimes() missing in libc.
 248                                         return os.stat(dest, follow_symlinks=False).st_mtime_ns
 249                                 else:
 250                                         return sstat.st_mtime_ns
 251                         else:
 252                                 # utime() in Python <3.3 only works on the target of a symlink, so it's not
 253                                 # possible to preserve mtime on symlinks.
 254                                 return os.lstat(dest)[stat.ST_MTIME]
 255                 except SystemExit as e:
 256                         raise
 257                 except Exception as e:
 258                         writemsg("!!! %s\n" % _("failed to properly create symlink:"),
 259                                 noiselevel=-1)
 260                         writemsg("!!! %s -> %s\n" % (dest, target), noiselevel=-1)
 261                         writemsg("!!! %s\n" % (e,), noiselevel=-1)
 262                         return None
 263
 264         hardlinked = False
 265         # Since identical files might be merged to multiple filesystems,
 266         # so os.link() calls might fail for some paths, so try them all.
 267         # For atomic replacement, first create the link as a temp file
 268         # and them use os.rename() to replace the destination.
 269         if hardlink_candidates:
 270                 head, tail = os.path.split(dest)
 271                 hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
 272                         (tail, os.getpid()))
 273                 try:
 274                         os.unlink(hardlink_tmp)
 275                 except OSError as e:
 276                         if e.errno != errno.ENOENT:
 277                                 writemsg(_("!!! Failed to remove hardlink temp file: %s\n") % \
 278                                         (hardlink_tmp,), noiselevel=-1)
 279                                 writemsg("!!! %s\n" % (e,), noiselevel=-1)
 280                                 return None
 281                         del e
 282                 for hardlink_src in hardlink_candidates:
 283                         try:
 284                                 os.link(hardlink_src, hardlink_tmp)
 285                         except OSError:
 286                                 continue
 287                         else:
 288                                 try:
 289                                         os.rename(hardlink_tmp, dest)
 290                                 except OSError as e:
 291                                         writemsg(_("!!! Failed to rename %s to %s\n") % \
 292                                                 (hardlink_tmp, dest), noiselevel=-1)
 293                                         writemsg("!!! %s\n" % (e,), noiselevel=-1)
 294                                         return None
 295                                 hardlinked = True
 296                                 try:
 297                                         _os.unlink(src_bytes)
 298                                 except OSError:
 299                                         pass
 300                                 break
 301
 302         renamefailed=1
 303         if hardlinked:
 304                 renamefailed = False
 305         if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev):
 306                 try:
 307                         if selinux_enabled:
 308                                 selinux.rename(src, dest)
 309                         else:
 310                                 os.rename(src,dest)
 311                         renamefailed=0
 312                 except OSError as e:
 313                         if e.errno != errno.EXDEV:
 314                                 # Some random error.
 315                                 writemsg("!!! %s\n" % _("Failed to move %(src)s to %(dest)s") %
 316                                         {"src": src, "dest": dest}, noiselevel=-1)
 317                                 writemsg("!!! %s\n" % (e,), noiselevel=-1)
 318                                 return None
 319                         # Invalid cross-device-link 'bind' mounted or actually Cross-Device
 320         if renamefailed:
 321                 if stat.S_ISREG(sstat[stat.ST_MODE]):
 322                         dest_tmp = dest + "#new"
 323                         dest_tmp_bytes = _unicode_encode(dest_tmp, encoding=encoding,
 324                                 errors='strict')
 325                         try: # For safety copy then move it over.
 326                                 _copyfile(src_bytes, dest_tmp_bytes)
 327                                 if xattr_enabled:
 328                                         try:
 329                                                 _copyxattr(src_bytes, dest_tmp_bytes,
 330                                                         exclude=mysettings.get("PORTAGE_XATTR_EXCLUDE", "security.* system.nfs4_acl"))
 331                                         except SystemExit:
 332                                                 raise
 333                                         except:
 334                                                 msg = _("Failed to copy extended attributes. "
 335                                                         "In order to avoid this error, set "
 336                                                         "FEATURES=\"-xattr\" in make.conf.")
 337                                                 msg = textwrap.wrap(msg, 65)
 338                                                 for line in msg:
 339                                                         writemsg("!!! %s\n" % (line,), noiselevel=-1)
 340                                                 raise
 341                                 _apply_stat(sstat, dest_tmp_bytes)
 342                                 _rename(dest_tmp_bytes, dest_bytes)
 343                                 _os.unlink(src_bytes)
 344                         except SystemExit as e:
 345                                 raise
 346                         except Exception as e:
 347                                 writemsg("!!! %s\n" % _('copy %(src)s -> %(dest)s failed.') %
 348                                         {"src": src, "dest": dest}, noiselevel=-1)
 349                                 writemsg("!!! %s\n" % (e,), noiselevel=-1)
 350                                 return None
 351                 else:
 352                         #we don't yet handle special, so we need to fall back to /bin/mv
 353                         a = spawn([MOVE_BINARY, '-f', src, dest], env=os.environ)
 354                         if a != os.EX_OK:
 355                                 writemsg(_("!!! Failed to move special file:\n"), noiselevel=-1)
 356                                 writemsg(_("!!! '%(src)s' to '%(dest)s'\n") % \
 357                                         {"src": _unicode_decode(src, encoding=encoding),
 358                                         "dest": _unicode_decode(dest, encoding=encoding)}, noiselevel=-1)
 359                                 writemsg("!!! %s\n" % a, noiselevel=-1)
 360                                 return None # failure
 361
 362         # In Python <3.3 always use stat_obj[stat.ST_MTIME] for the integral timestamp
 363         # which is returned, since the stat_obj.st_mtime float attribute rounds *up*
 364         # if the nanosecond part of the timestamp is 999999881 ns or greater.
 365         try:
 366                 if hardlinked:
 367                         if sys.hexversion >= 0x3030000:
 368                                 newmtime = os.stat(dest).st_mtime_ns
 369                         else:
 370                                 newmtime = os.stat(dest)[stat.ST_MTIME]
 371                 else:
 372                         # Note: It is not possible to preserve nanosecond precision
 373                         # (supported in POSIX.1-2008 via utimensat) with the IEEE 754
 374                         # double precision float which only has a 53 bit significand.
 375                         if newmtime is not None:
 376                                 if sys.hexversion >= 0x3030000:
 377                                         os.utime(dest, ns=(newmtime, newmtime))
 378                                 else:
 379                                         os.utime(dest, (newmtime, newmtime))
 380                         else:
 381                                 if sys.hexversion >= 0x3030000:
 382                                         newmtime = sstat.st_mtime_ns
 383                                 else:
 384                                         newmtime = sstat[stat.ST_MTIME]
 385                                 if renamefailed:
 386                                         if sys.hexversion >= 0x3030000:
 387                                                 # If rename succeeded then timestamps are automatically
 388                                                 # preserved with complete precision because the source
 389                                                 # and destination inodes are the same. Otherwise, manually
 390                                                 # update timestamps with nanosecond precision.
 391                                                 os.utime(dest, ns=(newmtime, newmtime))
 392                                         else:
 393                                                 # If rename succeeded then timestamps are automatically
 394                                                 # preserved with complete precision because the source
 395                                                 # and destination inodes are the same. Otherwise, round
 396                                                 # down to the nearest whole second since python's float
 397                                                 # st_mtime cannot be used to preserve the st_mtim.tv_nsec
 398                                                 # field with complete precision. Note that we have to use
 399                                                 # stat_obj[stat.ST_MTIME] here because the float
 400                                                 # stat_obj.st_mtime rounds *up* sometimes.
 401                                                 os.utime(dest, (newmtime, newmtime))
 402         except OSError:
 403                 # The utime can fail here with EPERM even though the move succeeded.
 404                 # Instead of failing, use stat to return the mtime if possible.
 405                 try:
 406                         if sys.hexversion >= 0x3030000:
 407                                 newmtime = os.stat(dest).st_mtime_ns
 408                         else:
 409                                 newmtime = os.stat(dest)[stat.ST_MTIME]
 410                 except OSError as e:
 411                         writemsg(_("!!! Failed to stat in movefile()\n"), noiselevel=-1)
 412                         writemsg("!!! %s\n" % dest, noiselevel=-1)
 413                         writemsg("!!! %s\n" % str(e), noiselevel=-1)
 414                         return None
 415
 416         if bsd_chflags:
 417                 # Restore the flags we saved before moving
 418                 if pflags:
 419                         bsd_chflags.chflags(os.path.dirname(dest), pflags)
 420
 421         return newmtime