pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004-2011 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   6         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   7         "unhardlink_lockfile", "hardlock_cleanup"]
   8
   9 import errno
  10 import fcntl
  11 import platform
  12 import sys
  13 import time
  14 import warnings
  15
  16 import portage
  17 from portage import os, _encodings, _unicode_decode
  18 from portage.exception import DirectoryNotFound, FileNotFound, \
  19         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  20 from portage.data import portage_gid
  21 from portage.util import writemsg
  22 from portage.localization import _
  23
  24 if sys.hexversion >= 0x3000000:
  25         basestring = str
  26
  27 HARDLINK_FD = -2
  28 _HARDLINK_POLL_LATENCY = 3 # seconds
  29 _default_lock_fn = fcntl.lockf
  30
  31 if platform.python_implementation() == 'PyPy':
  32         # workaround for https://bugs.pypy.org/issue747
  33         _default_lock_fn = fcntl.flock
  34
  35 # Used by emerge in order to disable the "waiting for lock" message
  36 # so that it doesn't interfere with the status display.
  37 _quiet = False
  38
  39 def lockdir(mydir, flags=0):
  40         return lockfile(mydir, wantnewlockfile=1, flags=flags)
  41 def unlockdir(mylock):
  42         return unlockfile(mylock)
  43
  44 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  45         waiting_msg=None, flags=0):
  46         """
  47         If wantnewlockfile is True then this creates a lockfile in the parent
  48         directory as the file: '.' + basename + '.portage_lockfile'.
  49         """
  50
  51         if not mypath:
  52                 raise InvalidData(_("Empty path given"))
  53
  54         # Support for file object or integer file descriptor parameters is
  55         # deprecated due to ambiguity in whether or not it's safe to close
  56         # the file descriptor, making it prone to "Bad file descriptor" errors
  57         # or file descriptor leaks.
  58         if isinstance(mypath, basestring) and mypath[-1] == '/':
  59                 mypath = mypath[:-1]
  60
  61         lockfilename_path = mypath
  62         if hasattr(mypath, 'fileno'):
  63                 warnings.warn("portage.locks.lockfile() support for "
  64                         "file object parameters is deprecated. Use a file path instead.",
  65                         DeprecationWarning, stacklevel=2)
  66                 lockfilename_path = getattr(mypath, 'name', None)
  67                 mypath = mypath.fileno()
  68         if isinstance(mypath, int):
  69                 warnings.warn("portage.locks.lockfile() support for integer file "
  70                         "descriptor parameters is deprecated. Use a file path instead.",
  71                         DeprecationWarning, stacklevel=2)
  72                 lockfilename    = mypath
  73                 wantnewlockfile = 0
  74                 unlinkfile      = 0
  75         elif wantnewlockfile:
  76                 base, tail = os.path.split(mypath)
  77                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  78                 lockfilename_path = lockfilename
  79                 unlinkfile   = 1
  80         else:
  81                 lockfilename = mypath
  82
  83         if isinstance(mypath, basestring):
  84                 if not os.path.exists(os.path.dirname(mypath)):
  85                         raise DirectoryNotFound(os.path.dirname(mypath))
  86                 preexisting = os.path.exists(lockfilename)
  87                 old_mask = os.umask(000)
  88                 try:
  89                         try:
  90                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
  91                         except OSError as e:
  92                                 func_call = "open('%s')" % lockfilename
  93                                 if e.errno == OperationNotPermitted.errno:
  94                                         raise OperationNotPermitted(func_call)
  95                                 elif e.errno == PermissionDenied.errno:
  96                                         raise PermissionDenied(func_call)
  97                                 else:
  98                                         raise
  99
 100                         if not preexisting:
 101                                 try:
 102                                         if os.stat(lockfilename).st_gid != portage_gid:
 103                                                 os.chown(lockfilename, -1, portage_gid)
 104                                 except OSError as e:
 105                                         if e.errno in (errno.ENOENT, errno.ESTALE):
 106                                                 return lockfile(mypath,
 107                                                         wantnewlockfile=wantnewlockfile,
 108                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
 109                                                         flags=flags)
 110                                         else:
 111                                                 writemsg("%s: chown('%s', -1, %d)\n" % \
 112                                                         (e, lockfilename, portage_gid), noiselevel=-1)
 113                                                 writemsg(_("Cannot chown a lockfile: '%s'\n") % \
 114                                                         lockfilename, noiselevel=-1)
 115                                                 writemsg(_("Group IDs of current user: %s\n") % \
 116                                                         " ".join(str(n) for n in os.getgroups()),
 117                                                         noiselevel=-1)
 118                 finally:
 119                         os.umask(old_mask)
 120
 121         elif isinstance(mypath, int):
 122                 myfd = mypath
 123
 124         else:
 125                 raise ValueError(_("Unknown type passed in '%s': '%s'") % \
 126                         (type(mypath), mypath))
 127
 128         # try for a non-blocking lock, if it's held, throw a message
 129         # we're waiting on lockfile and use a blocking attempt.
 130         locking_method = _default_lock_fn
 131         try:
 132                 if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
 133                         raise IOError(errno.ENOSYS, "Function not implemented")
 134                 locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
 135         except IOError as e:
 136                 if not hasattr(e, "errno"):
 137                         raise
 138                 if e.errno in (errno.EACCES, errno.EAGAIN):
 139                         # resource temp unavailable; eg, someone beat us to the lock.
 140                         if flags & os.O_NONBLOCK:
 141                                 os.close(myfd)
 142                                 raise TryAgain(mypath)
 143
 144                         global _quiet
 145                         if _quiet:
 146                                 out = None
 147                         else:
 148                                 out = portage.output.EOutput()
 149                         if waiting_msg is None:
 150                                 if isinstance(mypath, int):
 151                                         waiting_msg = _("waiting for lock on fd %i") % myfd
 152                                 else:
 153                                         waiting_msg = _("waiting for lock on %s\n") % lockfilename
 154                         if out is not None:
 155                                 out.ebegin(waiting_msg)
 156                         # try for the exclusive lock now.
 157                         try:
 158                                 locking_method(myfd, fcntl.LOCK_EX)
 159                         except EnvironmentError as e:
 160                                 if out is not None:
 161                                         out.eend(1, str(e))
 162                                 raise
 163                         if out is not None:
 164                                 out.eend(os.EX_OK)
 165                 elif e.errno in (errno.ENOSYS, errno.ENOLCK):
 166                         # We're not allowed to lock on this FS.
 167                         if not isinstance(lockfilename, int):
 168                                 # If a file object was passed in, it's not safe
 169                                 # to close the file descriptor because it may
 170                                 # still be in use.
 171                                 os.close(myfd)
 172                         lockfilename_path = _unicode_decode(lockfilename_path,
 173                                 encoding=_encodings['fs'], errors='strict')
 174                         if not isinstance(lockfilename_path, basestring):
 175                                 raise
 176                         link_success = hardlink_lockfile(lockfilename_path,
 177                                 waiting_msg=waiting_msg, flags=flags)
 178                         if not link_success:
 179                                 raise
 180                         lockfilename = lockfilename_path
 181                         locking_method = None
 182                         myfd = HARDLINK_FD
 183                 else:
 184                         raise
 185
 186
 187         if isinstance(lockfilename, basestring) and \
 188                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 189                 # The file was deleted on us... Keep trying to make one...
 190                 os.close(myfd)
 191                 writemsg(_("lockfile recurse\n"), 1)
 192                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 193                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 194                         waiting_msg=waiting_msg, flags=flags)
 195
 196         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 197         return (lockfilename,myfd,unlinkfile,locking_method)
 198
 199 def _fstat_nlink(fd):
 200         """
 201         @param fd: an open file descriptor
 202         @type fd: Integer
 203         @rtype: Integer
 204         @return: the current number of hardlinks to the file
 205         """
 206         try:
 207                 return os.fstat(fd).st_nlink
 208         except EnvironmentError as e:
 209                 if e.errno in (errno.ENOENT, errno.ESTALE):
 210                         # Some filesystems such as CIFS return
 211                         # ENOENT which means st_nlink == 0.
 212                         return 0
 213                 raise
 214
 215 def unlockfile(mytuple):
 216
 217         #XXX: Compatability hack.
 218         if len(mytuple) == 3:
 219                 lockfilename,myfd,unlinkfile = mytuple
 220                 locking_method = fcntl.flock
 221         elif len(mytuple) == 4:
 222                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 223         else:
 224                 raise InvalidData
 225
 226         if(myfd == HARDLINK_FD):
 227                 unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile)
 228                 return True
 229
 230         # myfd may be None here due to myfd = mypath in lockfile()
 231         if isinstance(lockfilename, basestring) and \
 232                 not os.path.exists(lockfilename):
 233                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 234                 if myfd is not None:
 235                         os.close(myfd)
 236                 return False
 237
 238         try:
 239                 if myfd is None:
 240                         myfd = os.open(lockfilename, os.O_WRONLY,0o660)
 241                         unlinkfile = 1
 242                 locking_method(myfd,fcntl.LOCK_UN)
 243         except OSError:
 244                 if isinstance(lockfilename, basestring):
 245                         os.close(myfd)
 246                 raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 247
 248         try:
 249                 # This sleep call was added to allow other processes that are
 250                 # waiting for a lock to be able to grab it before it is deleted.
 251                 # lockfile() already accounts for this situation, however, and
 252                 # the sleep here adds more time than is saved overall, so am
 253                 # commenting until it is proved necessary.
 254                 #time.sleep(0.0001)
 255                 if unlinkfile:
 256                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 257                         # We won the lock, so there isn't competition for it.
 258                         # We can safely delete the file.
 259                         writemsg(_("Got the lockfile...\n"), 1)
 260                         if _fstat_nlink(myfd) == 1:
 261                                 os.unlink(lockfilename)
 262                                 writemsg(_("Unlinked lockfile...\n"), 1)
 263                                 locking_method(myfd,fcntl.LOCK_UN)
 264                         else:
 265                                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 266                                 os.close(myfd)
 267                                 return False
 268         except SystemExit:
 269                 raise
 270         except Exception as e:
 271                 writemsg(_("Failed to get lock... someone took it.\n"), 1)
 272                 writemsg(str(e)+"\n",1)
 273
 274         # why test lockfilename?  because we may have been handed an
 275         # fd originally, and the caller might not like having their
 276         # open fd closed automatically on them.
 277         if isinstance(lockfilename, basestring):
 278                 os.close(myfd)
 279
 280         return True
 281
 282
 283
 284
 285 def hardlock_name(path):
 286         base, tail = os.path.split(path)
 287         return os.path.join(base, ".%s.hardlock-%s-%s" %
 288                 (tail, os.uname()[1], os.getpid()))
 289
 290 def hardlink_is_mine(link,lock):
 291         try:
 292                 lock_st = os.stat(lock)
 293                 if lock_st.st_nlink == 2:
 294                         link_st = os.stat(link)
 295                         return lock_st.st_ino == link_st.st_ino and \
 296                                 lock_st.st_dev == link_st.st_dev
 297         except OSError:
 298                 pass
 299         return False
 300
 301 def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
 302         waiting_msg=None, flags=0):
 303         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 304         We create a PRIVATE hardlink to the real lockfile, that is just a
 305         placeholder on the disk.
 306         If our file can 2 references, then we have the lock. :)
 307         Otherwise we lather, rise, and repeat.
 308         """
 309
 310         if max_wait is not DeprecationWarning:
 311                 warnings.warn("The 'max_wait' parameter of "
 312                         "portage.locks.hardlink_lockfile() is now unused. Use "
 313                         "flags=os.O_NONBLOCK instead.",
 314                         DeprecationWarning, stacklevel=2)
 315
 316         global _quiet
 317         out = None
 318         displayed_waiting_msg = False
 319         preexisting = os.path.exists(lockfilename)
 320         myhardlock = hardlock_name(lockfilename)
 321
 322         # myhardlock must not exist prior to our link() call, and we can
 323         # safely unlink it since its file name is unique to our PID
 324         try:
 325                 os.unlink(myhardlock)
 326         except OSError as e:
 327                 if e.errno in (errno.ENOENT, errno.ESTALE):
 328                         pass
 329                 else:
 330                         func_call = "unlink('%s')" % myhardlock
 331                         if e.errno == OperationNotPermitted.errno:
 332                                 raise OperationNotPermitted(func_call)
 333                         elif e.errno == PermissionDenied.errno:
 334                                 raise PermissionDenied(func_call)
 335                         else:
 336                                 raise
 337
 338         while True:
 339                 # create lockfilename if it doesn't exist yet
 340                 try:
 341                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
 342                 except OSError as e:
 343                         func_call = "open('%s')" % lockfilename
 344                         if e.errno == OperationNotPermitted.errno:
 345                                 raise OperationNotPermitted(func_call)
 346                         elif e.errno == PermissionDenied.errno:
 347                                 raise PermissionDenied(func_call)
 348                         else:
 349                                 raise
 350                 else:
 351                         myfd_st = None
 352                         try:
 353                                 myfd_st = os.fstat(myfd)
 354                                 if not preexisting:
 355                                         # Don't chown the file if it is preexisting, since we
 356                                         # want to preserve existing permissions in that case.
 357                                         if myfd_st.st_gid != portage_gid:
 358                                                 os.fchown(myfd, -1, portage_gid)
 359                         except OSError as e:
 360                                 if e.errno not in (errno.ENOENT, errno.ESTALE):
 361                                         writemsg("%s: fchown('%s', -1, %d)\n" % \
 362                                                 (e, lockfilename, portage_gid), noiselevel=-1)
 363                                         writemsg(_("Cannot chown a lockfile: '%s'\n") % \
 364                                                 lockfilename, noiselevel=-1)
 365                                         writemsg(_("Group IDs of current user: %s\n") % \
 366                                                 " ".join(str(n) for n in os.getgroups()),
 367                                                 noiselevel=-1)
 368                                 else:
 369                                         # another process has removed the file, so we'll have
 370                                         # to create it again
 371                                         continue
 372                         finally:
 373                                 os.close(myfd)
 374
 375                         # If fstat shows more than one hardlink, then it's extremely
 376                         # unlikely that the following link call will result in a lock,
 377                         # so optimize away the wasteful link call and sleep or raise
 378                         # TryAgain.
 379                         if myfd_st is not None and myfd_st.st_nlink < 2:
 380                                 try:
 381                                         os.link(lockfilename, myhardlock)
 382                                 except OSError as e:
 383                                         func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
 384                                         if e.errno == OperationNotPermitted.errno:
 385                                                 raise OperationNotPermitted(func_call)
 386                                         elif e.errno == PermissionDenied.errno:
 387                                                 raise PermissionDenied(func_call)
 388                                         elif e.errno in (errno.ESTALE, errno.ENOENT):
 389                                                 # another process has removed the file, so we'll have
 390                                                 # to create it again
 391                                                 continue
 392                                         else:
 393                                                 raise
 394                                 else:
 395                                         if hardlink_is_mine(myhardlock, lockfilename):
 396                                                 if out is not None:
 397                                                         out.eend(os.EX_OK)
 398                                                 break
 399
 400                                         try:
 401                                                 os.unlink(myhardlock)
 402                                         except OSError as e:
 403                                                 # This should not happen, since the file name of
 404                                                 # myhardlock is unique to our host and PID,
 405                                                 # and the above link() call succeeded.
 406                                                 if e.errno not in (errno.ENOENT, errno.ESTALE):
 407                                                         raise
 408                                                 raise FileNotFound(myhardlock)
 409
 410                 if flags & os.O_NONBLOCK:
 411                         raise TryAgain(lockfilename)
 412
 413                 if out is None and not _quiet:
 414                         out = portage.output.EOutput()
 415                 if out is not None and not displayed_waiting_msg:
 416                         displayed_waiting_msg = True
 417                         if waiting_msg is None:
 418                                 waiting_msg = _("waiting for lock on %s\n") % lockfilename
 419                         out.ebegin(waiting_msg)
 420
 421                 time.sleep(_HARDLINK_POLL_LATENCY)
 422
 423         return True
 424
 425 def unhardlink_lockfile(lockfilename, unlinkfile=True):
 426         myhardlock = hardlock_name(lockfilename)
 427         if unlinkfile and hardlink_is_mine(myhardlock, lockfilename):
 428                 # Make sure not to touch lockfilename unless we really have a lock.
 429                 try:
 430                         os.unlink(lockfilename)
 431                 except OSError:
 432                         pass
 433         try:
 434                 os.unlink(myhardlock)
 435         except OSError:
 436                 pass
 437
 438 def hardlock_cleanup(path, remove_all_locks=False):
 439         mypid  = str(os.getpid())
 440         myhost = os.uname()[1]
 441         mydl = os.listdir(path)
 442
 443         results = []
 444         mycount = 0
 445
 446         mylist = {}
 447         for x in mydl:
 448                 if os.path.isfile(path+"/"+x):
 449                         parts = x.split(".hardlock-")
 450                         if len(parts) == 2:
 451                                 filename = parts[0][1:]
 452                                 hostpid  = parts[1].split("-")
 453                                 host  = "-".join(hostpid[:-1])
 454                                 pid   = hostpid[-1]
 455
 456                                 if filename not in mylist:
 457                                         mylist[filename] = {}
 458                                 if host not in mylist[filename]:
 459                                         mylist[filename][host] = []
 460                                 mylist[filename][host].append(pid)
 461
 462                                 mycount += 1
 463
 464
 465         results.append(_("Found %(count)s locks") % {"count":mycount})
 466
 467         for x in mylist:
 468                 if myhost in mylist[x] or remove_all_locks:
 469                         mylockname = hardlock_name(path+"/"+x)
 470                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 471                            not os.path.exists(path+"/"+x) or \
 472                                  remove_all_locks:
 473                                 for y in mylist[x]:
 474                                         for z in mylist[x][y]:
 475                                                 filename = path+"/."+x+".hardlock-"+y+"-"+z
 476                                                 if filename == mylockname:
 477                                                         continue
 478                                                 try:
 479                                                         # We're sweeping through, unlinking everyone's locks.
 480                                                         os.unlink(filename)
 481                                                         results.append(_("Unlinked: ") + filename)
 482                                                 except OSError:
 483                                                         pass
 484                                 try:
 485                                         os.unlink(path+"/"+x)
 486                                         results.append(_("Unlinked: ") + path+"/"+x)
 487                                         os.unlink(mylockname)
 488                                         results.append(_("Unlinked: ") + mylockname)
 489                                 except OSError:
 490                                         pass
 491                         else:
 492                                 try:
 493                                         os.unlink(mylockname)
 494                                         results.append(_("Unlinked: ") + mylockname)
 495                                 except OSError:
 496                                         pass
 497
 498         return results
 499