pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004-2011 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   6         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   7         "unhardlink_lockfile", "hardlock_cleanup"]
   8
   9 import errno
  10 import fcntl
  11 import platform
  12 import sys
  13 import time
  14 import warnings
  15
  16 import portage
  17 from portage import os, _encodings, _unicode_decode
  18 from portage.exception import DirectoryNotFound, FileNotFound, \
  19         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  20 from portage.data import portage_gid
  21 from portage.util import writemsg
  22 from portage.localization import _
  23
  24 if sys.hexversion >= 0x3000000:
  25         basestring = str
  26
  27 HARDLINK_FD = -2
  28 _HARDLINK_POLL_LATENCY = 3 # seconds
  29 _default_lock_fn = fcntl.lockf
  30
  31 if platform.python_implementation() == 'PyPy':
  32         # workaround for https://bugs.pypy.org/issue747
  33         _default_lock_fn = fcntl.flock
  34
  35 # Used by emerge in order to disable the "waiting for lock" message
  36 # so that it doesn't interfere with the status display.
  37 _quiet = False
  38
  39 def lockdir(mydir, flags=0):
  40         return lockfile(mydir, wantnewlockfile=1, flags=flags)
  41 def unlockdir(mylock):
  42         return unlockfile(mylock)
  43
  44 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  45         waiting_msg=None, flags=0):
  46         """
  47         If wantnewlockfile is True then this creates a lockfile in the parent
  48         directory as the file: '.' + basename + '.portage_lockfile'.
  49         """
  50
  51         if not mypath:
  52                 raise InvalidData(_("Empty path given"))
  53
  54         if isinstance(mypath, basestring) and mypath[-1] == '/':
  55                 mypath = mypath[:-1]
  56
  57         lockfilename_path = mypath
  58         if hasattr(mypath, 'fileno'):
  59                 lockfilename_path = getattr(mypath, 'name', None)
  60                 mypath = mypath.fileno()
  61         if isinstance(mypath, int):
  62                 lockfilename    = mypath
  63                 wantnewlockfile = 0
  64                 unlinkfile      = 0
  65         elif wantnewlockfile:
  66                 base, tail = os.path.split(mypath)
  67                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  68                 lockfilename_path = lockfilename
  69                 unlinkfile   = 1
  70         else:
  71                 lockfilename = mypath
  72
  73         if isinstance(mypath, basestring):
  74                 if not os.path.exists(os.path.dirname(mypath)):
  75                         raise DirectoryNotFound(os.path.dirname(mypath))
  76                 preexisting = os.path.exists(lockfilename)
  77                 old_mask = os.umask(000)
  78                 try:
  79                         try:
  80                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
  81                         except OSError as e:
  82                                 func_call = "open('%s')" % lockfilename
  83                                 if e.errno == OperationNotPermitted.errno:
  84                                         raise OperationNotPermitted(func_call)
  85                                 elif e.errno == PermissionDenied.errno:
  86                                         raise PermissionDenied(func_call)
  87                                 else:
  88                                         raise
  89
  90                         if not preexisting:
  91                                 try:
  92                                         if os.stat(lockfilename).st_gid != portage_gid:
  93                                                 os.chown(lockfilename, -1, portage_gid)
  94                                 except OSError as e:
  95                                         if e.errno in (errno.ENOENT, errno.ESTALE):
  96                                                 return lockfile(mypath,
  97                                                         wantnewlockfile=wantnewlockfile,
  98                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
  99                                                         flags=flags)
 100                                         else:
 101                                                 writemsg("%s: chown('%s', -1, %d)\n" % \
 102                                                         (e, lockfilename, portage_gid), noiselevel=-1)
 103                                                 writemsg(_("Cannot chown a lockfile: '%s'\n") % \
 104                                                         lockfilename, noiselevel=-1)
 105                                                 writemsg(_("Group IDs of current user: %s\n") % \
 106                                                         " ".join(str(n) for n in os.getgroups()),
 107                                                         noiselevel=-1)
 108                 finally:
 109                         os.umask(old_mask)
 110
 111         elif isinstance(mypath, int):
 112                 myfd = mypath
 113
 114         else:
 115                 raise ValueError(_("Unknown type passed in '%s': '%s'") % \
 116                         (type(mypath), mypath))
 117
 118         # try for a non-blocking lock, if it's held, throw a message
 119         # we're waiting on lockfile and use a blocking attempt.
 120         locking_method = _default_lock_fn
 121         try:
 122                 if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
 123                         raise IOError(errno.ENOSYS, "Function not implemented")
 124                 locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
 125         except IOError as e:
 126                 if not hasattr(e, "errno"):
 127                         raise
 128                 if e.errno in (errno.EACCES, errno.EAGAIN):
 129                         # resource temp unavailable; eg, someone beat us to the lock.
 130                         if flags & os.O_NONBLOCK:
 131                                 os.close(myfd)
 132                                 raise TryAgain(mypath)
 133
 134                         global _quiet
 135                         if _quiet:
 136                                 out = None
 137                         else:
 138                                 out = portage.output.EOutput()
 139                         if waiting_msg is None:
 140                                 if isinstance(mypath, int):
 141                                         waiting_msg = _("waiting for lock on fd %i") % myfd
 142                                 else:
 143                                         waiting_msg = _("waiting for lock on %s\n") % lockfilename
 144                         if out is not None:
 145                                 out.ebegin(waiting_msg)
 146                         # try for the exclusive lock now.
 147                         try:
 148                                 locking_method(myfd, fcntl.LOCK_EX)
 149                         except EnvironmentError as e:
 150                                 if out is not None:
 151                                         out.eend(1, str(e))
 152                                 raise
 153                         if out is not None:
 154                                 out.eend(os.EX_OK)
 155                 elif e.errno in (errno.ENOSYS, errno.ENOLCK):
 156                         # We're not allowed to lock on this FS.
 157                         if not isinstance(lockfilename, int):
 158                                 # If a file object was passed in, it's not safe
 159                                 # to close the file descriptor because it may
 160                                 # still be in use (for example, see emergelog).
 161                                 os.close(myfd)
 162                         lockfilename_path = _unicode_decode(lockfilename_path,
 163                                 encoding=_encodings['fs'], errors='strict')
 164                         if not isinstance(lockfilename_path, basestring):
 165                                 raise
 166                         link_success = hardlink_lockfile(lockfilename_path,
 167                                 waiting_msg=waiting_msg, flags=flags)
 168                         if not link_success:
 169                                 raise
 170                         lockfilename = lockfilename_path
 171                         locking_method = None
 172                         myfd = HARDLINK_FD
 173                 else:
 174                         raise
 175
 176
 177         if isinstance(lockfilename, basestring) and \
 178                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 179                 # The file was deleted on us... Keep trying to make one...
 180                 os.close(myfd)
 181                 writemsg(_("lockfile recurse\n"), 1)
 182                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 183                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 184                         waiting_msg=waiting_msg, flags=flags)
 185
 186         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 187         return (lockfilename,myfd,unlinkfile,locking_method)
 188
 189 def _fstat_nlink(fd):
 190         """
 191         @param fd: an open file descriptor
 192         @type fd: Integer
 193         @rtype: Integer
 194         @return: the current number of hardlinks to the file
 195         """
 196         try:
 197                 return os.fstat(fd).st_nlink
 198         except EnvironmentError as e:
 199                 if e.errno in (errno.ENOENT, errno.ESTALE):
 200                         # Some filesystems such as CIFS return
 201                         # ENOENT which means st_nlink == 0.
 202                         return 0
 203                 raise
 204
 205 def unlockfile(mytuple):
 206
 207         #XXX: Compatability hack.
 208         if len(mytuple) == 3:
 209                 lockfilename,myfd,unlinkfile = mytuple
 210                 locking_method = fcntl.flock
 211         elif len(mytuple) == 4:
 212                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 213         else:
 214                 raise InvalidData
 215
 216         if(myfd == HARDLINK_FD):
 217                 unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile)
 218                 return True
 219
 220         # myfd may be None here due to myfd = mypath in lockfile()
 221         if isinstance(lockfilename, basestring) and \
 222                 not os.path.exists(lockfilename):
 223                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 224                 if myfd is not None:
 225                         os.close(myfd)
 226                 return False
 227
 228         try:
 229                 if myfd is None:
 230                         myfd = os.open(lockfilename, os.O_WRONLY,0o660)
 231                         unlinkfile = 1
 232                 locking_method(myfd,fcntl.LOCK_UN)
 233         except OSError:
 234                 if isinstance(lockfilename, basestring):
 235                         os.close(myfd)
 236                 raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 237
 238         try:
 239                 # This sleep call was added to allow other processes that are
 240                 # waiting for a lock to be able to grab it before it is deleted.
 241                 # lockfile() already accounts for this situation, however, and
 242                 # the sleep here adds more time than is saved overall, so am
 243                 # commenting until it is proved necessary.
 244                 #time.sleep(0.0001)
 245                 if unlinkfile:
 246                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 247                         # We won the lock, so there isn't competition for it.
 248                         # We can safely delete the file.
 249                         writemsg(_("Got the lockfile...\n"), 1)
 250                         if _fstat_nlink(myfd) == 1:
 251                                 os.unlink(lockfilename)
 252                                 writemsg(_("Unlinked lockfile...\n"), 1)
 253                                 locking_method(myfd,fcntl.LOCK_UN)
 254                         else:
 255                                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 256                                 os.close(myfd)
 257                                 return False
 258         except SystemExit:
 259                 raise
 260         except Exception as e:
 261                 writemsg(_("Failed to get lock... someone took it.\n"), 1)
 262                 writemsg(str(e)+"\n",1)
 263
 264         # why test lockfilename?  because we may have been handed an
 265         # fd originally, and the caller might not like having their
 266         # open fd closed automatically on them.
 267         if isinstance(lockfilename, basestring):
 268                 os.close(myfd)
 269
 270         return True
 271
 272
 273
 274
 275 def hardlock_name(path):
 276         base, tail = os.path.split(path)
 277         return os.path.join(base, ".%s.hardlock-%s-%s" %
 278                 (tail, os.uname()[1], os.getpid()))
 279
 280 def hardlink_is_mine(link,lock):
 281         try:
 282                 lock_st = os.stat(lock)
 283                 if lock_st.st_nlink == 2:
 284                         link_st = os.stat(link)
 285                         return lock_st.st_ino == link_st.st_ino and \
 286                                 lock_st.st_dev == link_st.st_dev
 287         except OSError:
 288                 return False
 289
 290 def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
 291         waiting_msg=None, flags=0):
 292         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 293         We create a PRIVATE hardlink to the real lockfile, that is just a
 294         placeholder on the disk.
 295         If our file can 2 references, then we have the lock. :)
 296         Otherwise we lather, rise, and repeat.
 297         """
 298
 299         if max_wait is not DeprecationWarning:
 300                 warnings.warn("The 'max_wait' parameter of "
 301                         "portage.locks.hardlink_lockfile() is now unused. Use "
 302                         "flags=os.O_NONBLOCK instead.",
 303                         DeprecationWarning, stacklevel=2)
 304
 305         global _quiet
 306         out = None
 307         displayed_waiting_msg = False
 308         myhardlock = hardlock_name(lockfilename)
 309
 310         # myhardlock must not exist prior to our link() call, and we can
 311         # can safely unlink it since its file name is unique to our PID
 312         try:
 313                 os.unlink(myhardlock)
 314         except OSError as e:
 315                 if e.errno in (errno.ENOENT, errno.ESTALE):
 316                         pass
 317                 else:
 318                         func_call = "unlink('%s')" % myhardlock
 319                         if e.errno == OperationNotPermitted.errno:
 320                                 raise OperationNotPermitted(func_call)
 321                         elif e.errno == PermissionDenied.errno:
 322                                 raise PermissionDenied(func_call)
 323                         else:
 324                                 raise
 325
 326         while True:
 327                 # create lockfilename if it doesn't exist yet
 328                 try:
 329                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
 330                 except OSError as e:
 331                         func_call = "open('%s')" % lockfilename
 332                         if e.errno == OperationNotPermitted.errno:
 333                                 raise OperationNotPermitted(func_call)
 334                         elif e.errno == PermissionDenied.errno:
 335                                 raise PermissionDenied(func_call)
 336                         else:
 337                                 raise
 338                 else:
 339                         try:
 340                                 if os.fstat(myfd).st_gid != portage_gid:
 341                                         os.fchown(myfd, -1, portage_gid)
 342                         except OSError as e:
 343                                 if e.errno not in (errno.ENOENT, errno.ESTALE):
 344                                         writemsg("%s: fchown('%s', -1, %d)\n" % \
 345                                                 (e, lockfilename, portage_gid), noiselevel=-1)
 346                                         writemsg(_("Cannot chown a lockfile: '%s'\n") % \
 347                                                 lockfilename, noiselevel=-1)
 348                                         writemsg(_("Group IDs of current user: %s\n") % \
 349                                                 " ".join(str(n) for n in os.getgroups()),
 350                                                 noiselevel=-1)
 351                         finally:
 352                                 os.close(myfd)
 353
 354                 try:
 355                         os.link(lockfilename, myhardlock)
 356                 except OSError as e:
 357                         func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
 358                         if e.errno == OperationNotPermitted.errno:
 359                                 raise OperationNotPermitted(func_call)
 360                         elif e.errno == PermissionDenied.errno:
 361                                 raise PermissionDenied(func_call)
 362                         elif e.errno in (errno.ESTALE, errno.ENOENT):
 363                                 # another process has removed the file, so we'll have
 364                                 # to create it again
 365                                 continue
 366                         else:
 367                                 raise
 368
 369                 if hardlink_is_mine(myhardlock, lockfilename):
 370                         if out is not None:
 371                                 out.eend(os.EX_OK)
 372                         break
 373
 374                 try:
 375                         os.unlink(myhardlock)
 376                 except OSError as e:
 377                         # This should not happen, since the file name of
 378                         # myhardlock is unique to our host and PID,
 379                         # and the above link() call succeeded.
 380                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 381                                 raise
 382                         raise FileNotFound(myhardlock)
 383
 384                 if flags & os.O_NONBLOCK:
 385                         raise TryAgain(lockfilename)
 386
 387                 if out is None and not _quiet:
 388                         out = portage.output.EOutput()
 389                 if out is not None and not displayed_waiting_msg:
 390                         displayed_waiting_msg = True
 391                         if waiting_msg is None:
 392                                 waiting_msg = _("waiting for lock on %s\n") % lockfilename
 393                         out.ebegin(waiting_msg)
 394
 395                 time.sleep(_HARDLINK_POLL_LATENCY)
 396
 397         return True
 398
 399 def unhardlink_lockfile(lockfilename, unlinkfile=True):
 400         myhardlock = hardlock_name(lockfilename)
 401         if unlinkfile and hardlink_is_mine(myhardlock, lockfilename):
 402                 # Make sure not to touch lockfilename unless we really have a lock.
 403                 try:
 404                         os.unlink(lockfilename)
 405                 except OSError:
 406                         pass
 407         try:
 408                 os.unlink(myhardlock)
 409         except OSError:
 410                 pass
 411
 412 def hardlock_cleanup(path, remove_all_locks=False):
 413         mypid  = str(os.getpid())
 414         myhost = os.uname()[1]
 415         mydl = os.listdir(path)
 416
 417         results = []
 418         mycount = 0
 419
 420         mylist = {}
 421         for x in mydl:
 422                 if os.path.isfile(path+"/"+x):
 423                         parts = x.split(".hardlock-")
 424                         if len(parts) == 2:
 425                                 filename = parts[0][1:]
 426                                 hostpid  = parts[1].split("-")
 427                                 host  = "-".join(hostpid[:-1])
 428                                 pid   = hostpid[-1]
 429
 430                                 if filename not in mylist:
 431                                         mylist[filename] = {}
 432                                 if host not in mylist[filename]:
 433                                         mylist[filename][host] = []
 434                                 mylist[filename][host].append(pid)
 435
 436                                 mycount += 1
 437
 438
 439         results.append(_("Found %(count)s locks") % {"count":mycount})
 440
 441         for x in mylist:
 442                 if myhost in mylist[x] or remove_all_locks:
 443                         mylockname = hardlock_name(path+"/"+x)
 444                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 445                            not os.path.exists(path+"/"+x) or \
 446                                  remove_all_locks:
 447                                 for y in mylist[x]:
 448                                         for z in mylist[x][y]:
 449                                                 filename = path+"/."+x+".hardlock-"+y+"-"+z
 450                                                 if filename == mylockname:
 451                                                         continue
 452                                                 try:
 453                                                         # We're sweeping through, unlinking everyone's locks.
 454                                                         os.unlink(filename)
 455                                                         results.append(_("Unlinked: ") + filename)
 456                                                 except OSError:
 457                                                         pass
 458                                 try:
 459                                         os.unlink(path+"/"+x)
 460                                         results.append(_("Unlinked: ") + path+"/"+x)
 461                                         os.unlink(mylockname)
 462                                         results.append(_("Unlinked: ") + mylockname)
 463                                 except OSError:
 464                                         pass
 465                         else:
 466                                 try:
 467                                         os.unlink(mylockname)
 468                                         results.append(_("Unlinked: ") + mylockname)
 469                                 except OSError:
 470                                         pass
 471
 472         return results
 473