pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004-2012 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   6         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   7         "unhardlink_lockfile", "hardlock_cleanup"]
   8
   9 import errno
  10 import fcntl
  11 import platform
  12 import sys
  13 import time
  14 import warnings
  15
  16 import portage
  17 from portage import os, _encodings, _unicode_decode
  18 from portage.exception import DirectoryNotFound, FileNotFound, \
  19         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  20 from portage.data import portage_gid
  21 from portage.util import writemsg
  22 from portage.localization import _
  23
  24 if sys.hexversion >= 0x3000000:
  25         basestring = str
  26
  27 HARDLINK_FD = -2
  28 _HARDLINK_POLL_LATENCY = 3 # seconds
  29 _default_lock_fn = fcntl.lockf
  30
  31 if platform.python_implementation() == 'PyPy':
  32         # workaround for https://bugs.pypy.org/issue747
  33         _default_lock_fn = fcntl.flock
  34
  35 # Used by emerge in order to disable the "waiting for lock" message
  36 # so that it doesn't interfere with the status display.
  37 _quiet = False
  38
  39
  40 _open_fds = set()
  41
  42 def _close_fds():
  43         """
  44         This is intended to be called after a fork, in order to close file
  45         descriptors for locks held by the parent process. This can be called
  46         safely after a fork without exec, unlike the _setup_pipes close_fds
  47         behavior.
  48         .
  49         """
  50         while _open_fds:
  51                 os.close(_open_fds.pop())
  52
  53 def lockdir(mydir, flags=0):
  54         return lockfile(mydir, wantnewlockfile=1, flags=flags)
  55 def unlockdir(mylock):
  56         return unlockfile(mylock)
  57
  58 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  59         waiting_msg=None, flags=0):
  60         """
  61         If wantnewlockfile is True then this creates a lockfile in the parent
  62         directory as the file: '.' + basename + '.portage_lockfile'.
  63         """
  64
  65         if not mypath:
  66                 raise InvalidData(_("Empty path given"))
  67
  68         # Support for file object or integer file descriptor parameters is
  69         # deprecated due to ambiguity in whether or not it's safe to close
  70         # the file descriptor, making it prone to "Bad file descriptor" errors
  71         # or file descriptor leaks.
  72         if isinstance(mypath, basestring) and mypath[-1] == '/':
  73                 mypath = mypath[:-1]
  74
  75         lockfilename_path = mypath
  76         if hasattr(mypath, 'fileno'):
  77                 warnings.warn("portage.locks.lockfile() support for "
  78                         "file object parameters is deprecated. Use a file path instead.",
  79                         DeprecationWarning, stacklevel=2)
  80                 lockfilename_path = getattr(mypath, 'name', None)
  81                 mypath = mypath.fileno()
  82         if isinstance(mypath, int):
  83                 warnings.warn("portage.locks.lockfile() support for integer file "
  84                         "descriptor parameters is deprecated. Use a file path instead.",
  85                         DeprecationWarning, stacklevel=2)
  86                 lockfilename    = mypath
  87                 wantnewlockfile = 0
  88                 unlinkfile      = 0
  89         elif wantnewlockfile:
  90                 base, tail = os.path.split(mypath)
  91                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  92                 lockfilename_path = lockfilename
  93                 unlinkfile   = 1
  94         else:
  95                 lockfilename = mypath
  96
  97         if isinstance(mypath, basestring):
  98                 if not os.path.exists(os.path.dirname(mypath)):
  99                         raise DirectoryNotFound(os.path.dirname(mypath))
 100                 preexisting = os.path.exists(lockfilename)
 101                 old_mask = os.umask(000)
 102                 try:
 103                         try:
 104                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
 105                         except OSError as e:
 106                                 func_call = "open('%s')" % lockfilename
 107                                 if e.errno == OperationNotPermitted.errno:
 108                                         raise OperationNotPermitted(func_call)
 109                                 elif e.errno == PermissionDenied.errno:
 110                                         raise PermissionDenied(func_call)
 111                                 else:
 112                                         raise
 113
 114                         if not preexisting:
 115                                 try:
 116                                         if os.stat(lockfilename).st_gid != portage_gid:
 117                                                 os.chown(lockfilename, -1, portage_gid)
 118                                 except OSError as e:
 119                                         if e.errno in (errno.ENOENT, errno.ESTALE):
 120                                                 return lockfile(mypath,
 121                                                         wantnewlockfile=wantnewlockfile,
 122                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
 123                                                         flags=flags)
 124                                         else:
 125                                                 writemsg("%s: chown('%s', -1, %d)\n" % \
 126                                                         (e, lockfilename, portage_gid), noiselevel=-1)
 127                                                 writemsg(_("Cannot chown a lockfile: '%s'\n") % \
 128                                                         lockfilename, noiselevel=-1)
 129                                                 writemsg(_("Group IDs of current user: %s\n") % \
 130                                                         " ".join(str(n) for n in os.getgroups()),
 131                                                         noiselevel=-1)
 132                 finally:
 133                         os.umask(old_mask)
 134
 135         elif isinstance(mypath, int):
 136                 myfd = mypath
 137
 138         else:
 139                 raise ValueError(_("Unknown type passed in '%s': '%s'") % \
 140                         (type(mypath), mypath))
 141
 142         # try for a non-blocking lock, if it's held, throw a message
 143         # we're waiting on lockfile and use a blocking attempt.
 144         locking_method = _default_lock_fn
 145         try:
 146                 if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
 147                         raise IOError(errno.ENOSYS, "Function not implemented")
 148                 locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
 149         except IOError as e:
 150                 if not hasattr(e, "errno"):
 151                         raise
 152                 if e.errno in (errno.EACCES, errno.EAGAIN):
 153                         # resource temp unavailable; eg, someone beat us to the lock.
 154                         if flags & os.O_NONBLOCK:
 155                                 os.close(myfd)
 156                                 raise TryAgain(mypath)
 157
 158                         global _quiet
 159                         if _quiet:
 160                                 out = None
 161                         else:
 162                                 out = portage.output.EOutput()
 163                         if waiting_msg is None:
 164                                 if isinstance(mypath, int):
 165                                         waiting_msg = _("waiting for lock on fd %i") % myfd
 166                                 else:
 167                                         waiting_msg = _("waiting for lock on %s\n") % lockfilename
 168                         if out is not None:
 169                                 out.ebegin(waiting_msg)
 170                         # try for the exclusive lock now.
 171                         try:
 172                                 locking_method(myfd, fcntl.LOCK_EX)
 173                         except EnvironmentError as e:
 174                                 if out is not None:
 175                                         out.eend(1, str(e))
 176                                 raise
 177                         if out is not None:
 178                                 out.eend(os.EX_OK)
 179                 elif e.errno in (errno.ENOSYS, errno.ENOLCK):
 180                         # We're not allowed to lock on this FS.
 181                         if not isinstance(lockfilename, int):
 182                                 # If a file object was passed in, it's not safe
 183                                 # to close the file descriptor because it may
 184                                 # still be in use.
 185                                 os.close(myfd)
 186                         lockfilename_path = _unicode_decode(lockfilename_path,
 187                                 encoding=_encodings['fs'], errors='strict')
 188                         if not isinstance(lockfilename_path, basestring):
 189                                 raise
 190                         link_success = hardlink_lockfile(lockfilename_path,
 191                                 waiting_msg=waiting_msg, flags=flags)
 192                         if not link_success:
 193                                 raise
 194                         lockfilename = lockfilename_path
 195                         locking_method = None
 196                         myfd = HARDLINK_FD
 197                 else:
 198                         raise
 199
 200
 201         if isinstance(lockfilename, basestring) and \
 202                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 203                 # The file was deleted on us... Keep trying to make one...
 204                 os.close(myfd)
 205                 writemsg(_("lockfile recurse\n"), 1)
 206                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 207                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 208                         waiting_msg=waiting_msg, flags=flags)
 209
 210         if myfd != HARDLINK_FD:
 211                 _open_fds.add(myfd)
 212
 213         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 214         return (lockfilename,myfd,unlinkfile,locking_method)
 215
 216 def _fstat_nlink(fd):
 217         """
 218         @param fd: an open file descriptor
 219         @type fd: Integer
 220         @rtype: Integer
 221         @return: the current number of hardlinks to the file
 222         """
 223         try:
 224                 return os.fstat(fd).st_nlink
 225         except EnvironmentError as e:
 226                 if e.errno in (errno.ENOENT, errno.ESTALE):
 227                         # Some filesystems such as CIFS return
 228                         # ENOENT which means st_nlink == 0.
 229                         return 0
 230                 raise
 231
 232 def unlockfile(mytuple):
 233
 234         #XXX: Compatability hack.
 235         if len(mytuple) == 3:
 236                 lockfilename,myfd,unlinkfile = mytuple
 237                 locking_method = fcntl.flock
 238         elif len(mytuple) == 4:
 239                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 240         else:
 241                 raise InvalidData
 242
 243         if(myfd == HARDLINK_FD):
 244                 unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile)
 245                 return True
 246
 247         # myfd may be None here due to myfd = mypath in lockfile()
 248         if isinstance(lockfilename, basestring) and \
 249                 not os.path.exists(lockfilename):
 250                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 251                 if myfd is not None:
 252                         os.close(myfd)
 253                         _open_fds.remove(myfd)
 254                 return False
 255
 256         try:
 257                 if myfd is None:
 258                         myfd = os.open(lockfilename, os.O_WRONLY,0o660)
 259                         unlinkfile = 1
 260                 locking_method(myfd,fcntl.LOCK_UN)
 261         except OSError:
 262                 if isinstance(lockfilename, basestring):
 263                         os.close(myfd)
 264                         _open_fds.remove(myfd)
 265                 raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 266
 267         try:
 268                 # This sleep call was added to allow other processes that are
 269                 # waiting for a lock to be able to grab it before it is deleted.
 270                 # lockfile() already accounts for this situation, however, and
 271                 # the sleep here adds more time than is saved overall, so am
 272                 # commenting until it is proved necessary.
 273                 #time.sleep(0.0001)
 274                 if unlinkfile:
 275                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 276                         # We won the lock, so there isn't competition for it.
 277                         # We can safely delete the file.
 278                         writemsg(_("Got the lockfile...\n"), 1)
 279                         if _fstat_nlink(myfd) == 1:
 280                                 os.unlink(lockfilename)
 281                                 writemsg(_("Unlinked lockfile...\n"), 1)
 282                                 locking_method(myfd,fcntl.LOCK_UN)
 283                         else:
 284                                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 285                                 os.close(myfd)
 286                                 _open_fds.remove(myfd)
 287                                 return False
 288         except SystemExit:
 289                 raise
 290         except Exception as e:
 291                 writemsg(_("Failed to get lock... someone took it.\n"), 1)
 292                 writemsg(str(e)+"\n",1)
 293
 294         # why test lockfilename?  because we may have been handed an
 295         # fd originally, and the caller might not like having their
 296         # open fd closed automatically on them.
 297         if isinstance(lockfilename, basestring):
 298                 os.close(myfd)
 299                 _open_fds.remove(myfd)
 300
 301         return True
 302
 303
 304
 305
 306 def hardlock_name(path):
 307         base, tail = os.path.split(path)
 308         return os.path.join(base, ".%s.hardlock-%s-%s" %
 309                 (tail, os.uname()[1], os.getpid()))
 310
 311 def hardlink_is_mine(link,lock):
 312         try:
 313                 lock_st = os.stat(lock)
 314                 if lock_st.st_nlink == 2:
 315                         link_st = os.stat(link)
 316                         return lock_st.st_ino == link_st.st_ino and \
 317                                 lock_st.st_dev == link_st.st_dev
 318         except OSError:
 319                 pass
 320         return False
 321
 322 def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
 323         waiting_msg=None, flags=0):
 324         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 325         We create a PRIVATE hardlink to the real lockfile, that is just a
 326         placeholder on the disk.
 327         If our file can 2 references, then we have the lock. :)
 328         Otherwise we lather, rise, and repeat.
 329         """
 330
 331         if max_wait is not DeprecationWarning:
 332                 warnings.warn("The 'max_wait' parameter of "
 333                         "portage.locks.hardlink_lockfile() is now unused. Use "
 334                         "flags=os.O_NONBLOCK instead.",
 335                         DeprecationWarning, stacklevel=2)
 336
 337         global _quiet
 338         out = None
 339         displayed_waiting_msg = False
 340         preexisting = os.path.exists(lockfilename)
 341         myhardlock = hardlock_name(lockfilename)
 342
 343         # myhardlock must not exist prior to our link() call, and we can
 344         # safely unlink it since its file name is unique to our PID
 345         try:
 346                 os.unlink(myhardlock)
 347         except OSError as e:
 348                 if e.errno in (errno.ENOENT, errno.ESTALE):
 349                         pass
 350                 else:
 351                         func_call = "unlink('%s')" % myhardlock
 352                         if e.errno == OperationNotPermitted.errno:
 353                                 raise OperationNotPermitted(func_call)
 354                         elif e.errno == PermissionDenied.errno:
 355                                 raise PermissionDenied(func_call)
 356                         else:
 357                                 raise
 358
 359         while True:
 360                 # create lockfilename if it doesn't exist yet
 361                 try:
 362                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
 363                 except OSError as e:
 364                         func_call = "open('%s')" % lockfilename
 365                         if e.errno == OperationNotPermitted.errno:
 366                                 raise OperationNotPermitted(func_call)
 367                         elif e.errno == PermissionDenied.errno:
 368                                 raise PermissionDenied(func_call)
 369                         else:
 370                                 raise
 371                 else:
 372                         myfd_st = None
 373                         try:
 374                                 myfd_st = os.fstat(myfd)
 375                                 if not preexisting:
 376                                         # Don't chown the file if it is preexisting, since we
 377                                         # want to preserve existing permissions in that case.
 378                                         if myfd_st.st_gid != portage_gid:
 379                                                 os.fchown(myfd, -1, portage_gid)
 380                         except OSError as e:
 381                                 if e.errno not in (errno.ENOENT, errno.ESTALE):
 382                                         writemsg("%s: fchown('%s', -1, %d)\n" % \
 383                                                 (e, lockfilename, portage_gid), noiselevel=-1)
 384                                         writemsg(_("Cannot chown a lockfile: '%s'\n") % \
 385                                                 lockfilename, noiselevel=-1)
 386                                         writemsg(_("Group IDs of current user: %s\n") % \
 387                                                 " ".join(str(n) for n in os.getgroups()),
 388                                                 noiselevel=-1)
 389                                 else:
 390                                         # another process has removed the file, so we'll have
 391                                         # to create it again
 392                                         continue
 393                         finally:
 394                                 os.close(myfd)
 395
 396                         # If fstat shows more than one hardlink, then it's extremely
 397                         # unlikely that the following link call will result in a lock,
 398                         # so optimize away the wasteful link call and sleep or raise
 399                         # TryAgain.
 400                         if myfd_st is not None and myfd_st.st_nlink < 2:
 401                                 try:
 402                                         os.link(lockfilename, myhardlock)
 403                                 except OSError as e:
 404                                         func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
 405                                         if e.errno == OperationNotPermitted.errno:
 406                                                 raise OperationNotPermitted(func_call)
 407                                         elif e.errno == PermissionDenied.errno:
 408                                                 raise PermissionDenied(func_call)
 409                                         elif e.errno in (errno.ESTALE, errno.ENOENT):
 410                                                 # another process has removed the file, so we'll have
 411                                                 # to create it again
 412                                                 continue
 413                                         else:
 414                                                 raise
 415                                 else:
 416                                         if hardlink_is_mine(myhardlock, lockfilename):
 417                                                 if out is not None:
 418                                                         out.eend(os.EX_OK)
 419                                                 break
 420
 421                                         try:
 422                                                 os.unlink(myhardlock)
 423                                         except OSError as e:
 424                                                 # This should not happen, since the file name of
 425                                                 # myhardlock is unique to our host and PID,
 426                                                 # and the above link() call succeeded.
 427                                                 if e.errno not in (errno.ENOENT, errno.ESTALE):
 428                                                         raise
 429                                                 raise FileNotFound(myhardlock)
 430
 431                 if flags & os.O_NONBLOCK:
 432                         raise TryAgain(lockfilename)
 433
 434                 if out is None and not _quiet:
 435                         out = portage.output.EOutput()
 436                 if out is not None and not displayed_waiting_msg:
 437                         displayed_waiting_msg = True
 438                         if waiting_msg is None:
 439                                 waiting_msg = _("waiting for lock on %s\n") % lockfilename
 440                         out.ebegin(waiting_msg)
 441
 442                 time.sleep(_HARDLINK_POLL_LATENCY)
 443
 444         return True
 445
 446 def unhardlink_lockfile(lockfilename, unlinkfile=True):
 447         myhardlock = hardlock_name(lockfilename)
 448         if unlinkfile and hardlink_is_mine(myhardlock, lockfilename):
 449                 # Make sure not to touch lockfilename unless we really have a lock.
 450                 try:
 451                         os.unlink(lockfilename)
 452                 except OSError:
 453                         pass
 454         try:
 455                 os.unlink(myhardlock)
 456         except OSError:
 457                 pass
 458
 459 def hardlock_cleanup(path, remove_all_locks=False):
 460         mypid  = str(os.getpid())
 461         myhost = os.uname()[1]
 462         mydl = os.listdir(path)
 463
 464         results = []
 465         mycount = 0
 466
 467         mylist = {}
 468         for x in mydl:
 469                 if os.path.isfile(path+"/"+x):
 470                         parts = x.split(".hardlock-")
 471                         if len(parts) == 2:
 472                                 filename = parts[0][1:]
 473                                 hostpid  = parts[1].split("-")
 474                                 host  = "-".join(hostpid[:-1])
 475                                 pid   = hostpid[-1]
 476
 477                                 if filename not in mylist:
 478                                         mylist[filename] = {}
 479                                 if host not in mylist[filename]:
 480                                         mylist[filename][host] = []
 481                                 mylist[filename][host].append(pid)
 482
 483                                 mycount += 1
 484
 485
 486         results.append(_("Found %(count)s locks") % {"count":mycount})
 487
 488         for x in mylist:
 489                 if myhost in mylist[x] or remove_all_locks:
 490                         mylockname = hardlock_name(path+"/"+x)
 491                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 492                            not os.path.exists(path+"/"+x) or \
 493                                  remove_all_locks:
 494                                 for y in mylist[x]:
 495                                         for z in mylist[x][y]:
 496                                                 filename = path+"/."+x+".hardlock-"+y+"-"+z
 497                                                 if filename == mylockname:
 498                                                         continue
 499                                                 try:
 500                                                         # We're sweeping through, unlinking everyone's locks.
 501                                                         os.unlink(filename)
 502                                                         results.append(_("Unlinked: ") + filename)
 503                                                 except OSError:
 504                                                         pass
 505                                 try:
 506                                         os.unlink(path+"/"+x)
 507                                         results.append(_("Unlinked: ") + path+"/"+x)
 508                                         os.unlink(mylockname)
 509                                         results.append(_("Unlinked: ") + mylockname)
 510                                 except OSError:
 511                                         pass
 512                         else:
 513                                 try:
 514                                         os.unlink(mylockname)
 515                                         results.append(_("Unlinked: ") + mylockname)
 516                                 except OSError:
 517                                         pass
 518
 519         return results
 520