pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004-2010 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   6         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   7         "unhardlink_lockfile", "hardlock_cleanup"]
   8
   9 import errno
  10 import fcntl
  11 import platform
  12 import stat
  13 import sys
  14 import time
  15
  16 import portage
  17 from portage import os
  18 from portage.const import PORTAGE_BIN_PATH
  19 from portage.exception import DirectoryNotFound, FileNotFound, \
  20         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  21 from portage.data import portage_gid
  22 from portage.util import writemsg
  23 from portage.localization import _
  24
  25 if sys.hexversion >= 0x3000000:
  26         basestring = str
  27
  28 HARDLINK_FD = -2
  29 _default_lock_fn = fcntl.lockf
  30
  31 if platform.python_implementation() == 'PyPy':
  32         # workaround for https://bugs.pypy.org/issue747
  33         _default_lock_fn = fcntl.flock
  34
  35 # Used by emerge in order to disable the "waiting for lock" message
  36 # so that it doesn't interfere with the status display.
  37 _quiet = False
  38
  39 def lockdir(mydir, flags=0):
  40         return lockfile(mydir, wantnewlockfile=1, flags=flags)
  41 def unlockdir(mylock):
  42         return unlockfile(mylock)
  43
  44 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  45         waiting_msg=None, flags=0):
  46         """
  47         If wantnewlockfile is True then this creates a lockfile in the parent
  48         directory as the file: '.' + basename + '.portage_lockfile'.
  49         """
  50
  51         if not mypath:
  52                 raise InvalidData(_("Empty path given"))
  53
  54         if isinstance(mypath, basestring) and mypath[-1] == '/':
  55                 mypath = mypath[:-1]
  56
  57         if hasattr(mypath, 'fileno'):
  58                 mypath = mypath.fileno()
  59         if isinstance(mypath, int):
  60                 lockfilename    = mypath
  61                 wantnewlockfile = 0
  62                 unlinkfile      = 0
  63         elif wantnewlockfile:
  64                 base, tail = os.path.split(mypath)
  65                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  66                 del base, tail
  67                 unlinkfile   = 1
  68         else:
  69                 lockfilename = mypath
  70
  71         if isinstance(mypath, basestring):
  72                 if not os.path.exists(os.path.dirname(mypath)):
  73                         raise DirectoryNotFound(os.path.dirname(mypath))
  74                 preexisting = os.path.exists(lockfilename)
  75                 old_mask = os.umask(000)
  76                 try:
  77                         try:
  78                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
  79                         except OSError as e:
  80                                 func_call = "open('%s')" % lockfilename
  81                                 if e.errno == OperationNotPermitted.errno:
  82                                         raise OperationNotPermitted(func_call)
  83                                 elif e.errno == PermissionDenied.errno:
  84                                         raise PermissionDenied(func_call)
  85                                 else:
  86                                         raise
  87
  88                         if not preexisting:
  89                                 try:
  90                                         if os.stat(lockfilename).st_gid != portage_gid:
  91                                                 os.chown(lockfilename, -1, portage_gid)
  92                                 except OSError as e:
  93                                         if e.errno in (errno.ENOENT, errno.ESTALE):
  94                                                 return lockfile(mypath,
  95                                                         wantnewlockfile=wantnewlockfile,
  96                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
  97                                                         flags=flags)
  98                                         else:
  99                                                 writemsg("%s: chown('%s', -1, %d)\n" % \
 100                                                         (e, lockfilename, portage_gid), noiselevel=-1)
 101                                                 writemsg(_("Cannot chown a lockfile: '%s'\n") % \
 102                                                         lockfilename, noiselevel=-1)
 103                                                 writemsg(_("Group IDs of current user: %s\n") % \
 104                                                         " ".join(str(n) for n in os.getgroups()),
 105                                                         noiselevel=-1)
 106                 finally:
 107                         os.umask(old_mask)
 108
 109         elif isinstance(mypath, int):
 110                 myfd = mypath
 111
 112         else:
 113                 raise ValueError(_("Unknown type passed in '%s': '%s'") % \
 114                         (type(mypath), mypath))
 115
 116         # try for a non-blocking lock, if it's held, throw a message
 117         # we're waiting on lockfile and use a blocking attempt.
 118         locking_method = _default_lock_fn
 119         try:
 120                 locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
 121         except IOError as e:
 122                 if not hasattr(e, "errno"):
 123                         raise
 124                 if e.errno in (errno.EACCES, errno.EAGAIN):
 125                         # resource temp unavailable; eg, someone beat us to the lock.
 126                         if flags & os.O_NONBLOCK:
 127                                 os.close(myfd)
 128                                 raise TryAgain(mypath)
 129
 130                         global _quiet
 131                         if _quiet:
 132                                 out = None
 133                         else:
 134                                 out = portage.output.EOutput()
 135                         if waiting_msg is None:
 136                                 if isinstance(mypath, int):
 137                                         waiting_msg = _("waiting for lock on fd %i") % myfd
 138                                 else:
 139                                         waiting_msg = _("waiting for lock on %s\n") % lockfilename
 140                         if out is not None:
 141                                 out.ebegin(waiting_msg)
 142                         # try for the exclusive lock now.
 143                         try:
 144                                 locking_method(myfd, fcntl.LOCK_EX)
 145                         except EnvironmentError as e:
 146                                 if out is not None:
 147                                         out.eend(1, str(e))
 148                                 raise
 149                         if out is not None:
 150                                 out.eend(os.EX_OK)
 151                 elif e.errno == errno.ENOLCK:
 152                         # We're not allowed to lock on this FS.
 153                         os.close(myfd)
 154                         link_success = False
 155                         if lockfilename == str(lockfilename):
 156                                 if wantnewlockfile:
 157                                         try:
 158                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
 159                                                         os.unlink(lockfilename)
 160                                         except OSError:
 161                                                 pass
 162                                         link_success = hardlink_lockfile(lockfilename)
 163                         if not link_success:
 164                                 raise
 165                         locking_method = None
 166                         myfd = HARDLINK_FD
 167                 else:
 168                         raise
 169
 170
 171         if isinstance(lockfilename, basestring) and \
 172                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 173                 # The file was deleted on us... Keep trying to make one...
 174                 os.close(myfd)
 175                 writemsg(_("lockfile recurse\n"), 1)
 176                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 177                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 178                         waiting_msg=waiting_msg, flags=flags)
 179
 180         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 181         return (lockfilename,myfd,unlinkfile,locking_method)
 182
 183 def _fstat_nlink(fd):
 184         """
 185         @param fd: an open file descriptor
 186         @type fd: Integer
 187         @rtype: Integer
 188         @return: the current number of hardlinks to the file
 189         """
 190         try:
 191                 return os.fstat(fd).st_nlink
 192         except EnvironmentError as e:
 193                 if e.errno in (errno.ENOENT, errno.ESTALE):
 194                         # Some filesystems such as CIFS return
 195                         # ENOENT which means st_nlink == 0.
 196                         return 0
 197                 raise
 198
 199 def unlockfile(mytuple):
 200
 201         #XXX: Compatability hack.
 202         if len(mytuple) == 3:
 203                 lockfilename,myfd,unlinkfile = mytuple
 204                 locking_method = fcntl.flock
 205         elif len(mytuple) == 4:
 206                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 207         else:
 208                 raise InvalidData
 209
 210         if(myfd == HARDLINK_FD):
 211                 unhardlink_lockfile(lockfilename)
 212                 return True
 213
 214         # myfd may be None here due to myfd = mypath in lockfile()
 215         if isinstance(lockfilename, basestring) and \
 216                 not os.path.exists(lockfilename):
 217                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 218                 if myfd is not None:
 219                         os.close(myfd)
 220                 return False
 221
 222         try:
 223                 if myfd is None:
 224                         myfd = os.open(lockfilename, os.O_WRONLY,0o660)
 225                         unlinkfile = 1
 226                 locking_method(myfd,fcntl.LOCK_UN)
 227         except OSError:
 228                 if isinstance(lockfilename, basestring):
 229                         os.close(myfd)
 230                 raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 231
 232         try:
 233                 # This sleep call was added to allow other processes that are
 234                 # waiting for a lock to be able to grab it before it is deleted.
 235                 # lockfile() already accounts for this situation, however, and
 236                 # the sleep here adds more time than is saved overall, so am
 237                 # commenting until it is proved necessary.
 238                 #time.sleep(0.0001)
 239                 if unlinkfile:
 240                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 241                         # We won the lock, so there isn't competition for it.
 242                         # We can safely delete the file.
 243                         writemsg(_("Got the lockfile...\n"), 1)
 244                         if _fstat_nlink(myfd) == 1:
 245                                 os.unlink(lockfilename)
 246                                 writemsg(_("Unlinked lockfile...\n"), 1)
 247                                 locking_method(myfd,fcntl.LOCK_UN)
 248                         else:
 249                                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 250                                 os.close(myfd)
 251                                 return False
 252         except SystemExit:
 253                 raise
 254         except Exception as e:
 255                 writemsg(_("Failed to get lock... someone took it.\n"), 1)
 256                 writemsg(str(e)+"\n",1)
 257
 258         # why test lockfilename?  because we may have been handed an
 259         # fd originally, and the caller might not like having their
 260         # open fd closed automatically on them.
 261         if isinstance(lockfilename, basestring):
 262                 os.close(myfd)
 263
 264         return True
 265
 266
 267
 268
 269 def hardlock_name(path):
 270         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 271
 272 def hardlink_is_mine(link,lock):
 273         try:
 274                 return os.stat(link).st_nlink == 2
 275         except OSError:
 276                 return False
 277
 278 def hardlink_lockfile(lockfilename, max_wait=14400):
 279         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 280         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 281         Then we HARDLINK the real lockfile to that private file.
 282         If our file can 2 references, then we have the lock. :)
 283         Otherwise we lather, rise, and repeat.
 284         We default to a 4 hour timeout.
 285         """
 286
 287         start_time = time.time()
 288         myhardlock = hardlock_name(lockfilename)
 289         reported_waiting = False
 290
 291         while(time.time() < (start_time + max_wait)):
 292                 # We only need it to exist.
 293                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0o660)
 294                 os.close(myfd)
 295
 296                 if not os.path.exists(myhardlock):
 297                         raise FileNotFound(
 298                                 _("Created lockfile is missing: %(filename)s") % \
 299                                 {"filename" : myhardlock})
 300
 301                 try:
 302                         res = os.link(myhardlock, lockfilename)
 303                 except OSError:
 304                         pass
 305
 306                 if hardlink_is_mine(myhardlock, lockfilename):
 307                         # We have the lock.
 308                         if reported_waiting:
 309                                 writemsg("\n", noiselevel=-1)
 310                         return True
 311
 312                 if reported_waiting:
 313                         writemsg(".", noiselevel=-1)
 314                 else:
 315                         reported_waiting = True
 316                         msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
 317                                 "%(bin_path)s/clean_locks can fix stuck locks.\n"
 318                                 "Lockfile: %(lockfilename)s\n") % \
 319                                 {"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
 320                         writemsg(msg, noiselevel=-1)
 321                 time.sleep(3)
 322
 323         os.unlink(myhardlock)
 324         return False
 325
 326 def unhardlink_lockfile(lockfilename):
 327         myhardlock = hardlock_name(lockfilename)
 328         if hardlink_is_mine(myhardlock, lockfilename):
 329                 # Make sure not to touch lockfilename unless we really have a lock.
 330                 try:
 331                         os.unlink(lockfilename)
 332                 except OSError:
 333                         pass
 334         try:
 335                 os.unlink(myhardlock)
 336         except OSError:
 337                 pass
 338
 339 def hardlock_cleanup(path, remove_all_locks=False):
 340         mypid  = str(os.getpid())
 341         myhost = os.uname()[1]
 342         mydl = os.listdir(path)
 343
 344         results = []
 345         mycount = 0
 346
 347         mylist = {}
 348         for x in mydl:
 349                 if os.path.isfile(path+"/"+x):
 350                         parts = x.split(".hardlock-")
 351                         if len(parts) == 2:
 352                                 filename = parts[0]
 353                                 hostpid  = parts[1].split("-")
 354                                 host  = "-".join(hostpid[:-1])
 355                                 pid   = hostpid[-1]
 356
 357                                 if filename not in mylist:
 358                                         mylist[filename] = {}
 359                                 if host not in mylist[filename]:
 360                                         mylist[filename][host] = []
 361                                 mylist[filename][host].append(pid)
 362
 363                                 mycount += 1
 364
 365
 366         results.append(_("Found %(count)s locks") % {"count":mycount})
 367
 368         for x in mylist:
 369                 if myhost in mylist[x] or remove_all_locks:
 370                         mylockname = hardlock_name(path+"/"+x)
 371                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 372                            not os.path.exists(path+"/"+x) or \
 373                                  remove_all_locks:
 374                                 for y in mylist[x]:
 375                                         for z in mylist[x][y]:
 376                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 377                                                 if filename == mylockname:
 378                                                         continue
 379                                                 try:
 380                                                         # We're sweeping through, unlinking everyone's locks.
 381                                                         os.unlink(filename)
 382                                                         results.append(_("Unlinked: ") + filename)
 383                                                 except OSError:
 384                                                         pass
 385                                 try:
 386                                         os.unlink(path+"/"+x)
 387                                         results.append(_("Unlinked: ") + path+"/"+x)
 388                                         os.unlink(mylockname)
 389                                         results.append(_("Unlinked: ") + mylockname)
 390                                 except OSError:
 391                                         pass
 392                         else:
 393                                 try:
 394                                         os.unlink(mylockname)
 395                                         results.append(_("Unlinked: ") + mylockname)
 396                                 except OSError:
 397                                         pass
 398
 399         return results
 400