pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   7         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   8         "unhardlink_lockfile", "hardlock_cleanup"]
   9
  10 import errno
  11 import stat
  12 import time
  13 from portage import os
  14 from portage.exception import DirectoryNotFound, FileNotFound, \
  15         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  16 from portage.data import portage_gid
  17 from portage.output import EOutput
  18 from portage.util import writemsg
  19 from portage.localization import _
  20
  21 HARDLINK_FD = -2
  22
  23 # Used by emerge in order to disable the "waiting for lock" message
  24 # so that it doesn't interfere with the status display.
  25 _quiet = False
  26
  27 def lockdir(mydir):
  28         return lockfile(mydir,wantnewlockfile=1)
  29 def unlockdir(mylock):
  30         return unlockfile(mylock)
  31
  32 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  33         waiting_msg=None, flags=0):
  34         """
  35         If wantnewlockfile is True then this creates a lockfile in the parent
  36         directory as the file: '.' + basename + '.portage_lockfile'.
  37         """
  38         import fcntl
  39
  40         if not mypath:
  41                 raise InvalidData(_("Empty path given"))
  42
  43         if isinstance(mypath, basestring) and mypath[-1] == '/':
  44                 mypath = mypath[:-1]
  45
  46         if hasattr(mypath, 'fileno'):
  47                 mypath = mypath.fileno()
  48         if isinstance(mypath, int):
  49                 lockfilename    = mypath
  50                 wantnewlockfile = 0
  51                 unlinkfile      = 0
  52         elif wantnewlockfile:
  53                 base, tail = os.path.split(mypath)
  54                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  55                 del base, tail
  56                 unlinkfile   = 1
  57         else:
  58                 lockfilename = mypath
  59
  60         if isinstance(mypath, basestring):
  61                 if not os.path.exists(os.path.dirname(mypath)):
  62                         raise DirectoryNotFound(os.path.dirname(mypath))
  63                 preexisting = os.path.exists(lockfilename)
  64                 old_mask = os.umask(000)
  65                 try:
  66                         try:
  67                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0660)
  68                         except OSError as e:
  69                                 func_call = "open('%s')" % lockfilename
  70                                 if e.errno == OperationNotPermitted.errno:
  71                                         raise OperationNotPermitted(func_call)
  72                                 elif e.errno == PermissionDenied.errno:
  73                                         raise PermissionDenied(func_call)
  74                                 else:
  75                                         raise
  76
  77                         if not preexisting:
  78                                 try:
  79                                         if os.stat(lockfilename).st_gid != portage_gid:
  80                                                 os.chown(lockfilename, -1, portage_gid)
  81                                 except OSError as e:
  82                                         if e.errno in (errno.ENOENT, errno.ESTALE):
  83                                                 return lockfile(mypath,
  84                                                         wantnewlockfile=wantnewlockfile,
  85                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
  86                                                         flags=flags)
  87                                         else:
  88                                                 writemsg(_("Cannot chown a lockfile. This could "
  89                                                         "cause inconvenience later.\n"))
  90
  91                 finally:
  92                         os.umask(old_mask)
  93
  94         elif isinstance(mypath, int):
  95                 myfd = mypath
  96
  97         else:
  98                 raise ValueError(_("Unknown type passed in '%s': '%s'") % \
  99                         (type(mypath), mypath))
 100
 101         # try for a non-blocking lock, if it's held, throw a message
 102         # we're waiting on lockfile and use a blocking attempt.
 103         locking_method = fcntl.lockf
 104         try:
 105                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 106         except IOError as e:
 107                 if "errno" not in dir(e):
 108                         raise
 109                 if e.errno in (errno.EACCES, errno.EAGAIN):
 110                         # resource temp unavailable; eg, someone beat us to the lock.
 111                         if flags & os.O_NONBLOCK:
 112                                 raise TryAgain(mypath)
 113
 114                         global _quiet
 115                         out = EOutput()
 116                         out.quiet = _quiet
 117                         if waiting_msg is None:
 118                                 if isinstance(mypath, int):
 119                                         waiting_msg = _("waiting for lock on fd %i") % myfd
 120                                 else:
 121                                         waiting_msg = _("waiting for lock on %s\n") % lockfilename
 122                         out.ebegin(waiting_msg)
 123                         # try for the exclusive lock now.
 124                         try:
 125                                 fcntl.lockf(myfd, fcntl.LOCK_EX)
 126                         except EnvironmentError as e:
 127                                 out.eend(1, str(e))
 128                                 raise
 129                         out.eend(os.EX_OK)
 130                 elif e.errno == errno.ENOLCK:
 131                         # We're not allowed to lock on this FS.
 132                         os.close(myfd)
 133                         link_success = False
 134                         if lockfilename == str(lockfilename):
 135                                 if wantnewlockfile:
 136                                         try:
 137                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
 138                                                         os.unlink(lockfilename)
 139                                         except OSError:
 140                                                 pass
 141                                         link_success = hardlink_lockfile(lockfilename)
 142                         if not link_success:
 143                                 raise
 144                         locking_method = None
 145                         myfd = HARDLINK_FD
 146                 else:
 147                         raise
 148
 149
 150         if isinstance(lockfilename, basestring) and \
 151                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 152                 # The file was deleted on us... Keep trying to make one...
 153                 os.close(myfd)
 154                 writemsg(_("lockfile recurse\n"), 1)
 155                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 156                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 157                         waiting_msg=waiting_msg, flags=flags)
 158
 159         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 160         return (lockfilename,myfd,unlinkfile,locking_method)
 161
 162 def _fstat_nlink(fd):
 163         """
 164         @param fd: an open file descriptor
 165         @type fd: Integer
 166         @rtype: Integer
 167         @return: the current number of hardlinks to the file
 168         """
 169         try:
 170                 return os.fstat(fd).st_nlink
 171         except EnvironmentError as e:
 172                 if e.errno in (errno.ENOENT, errno.ESTALE):
 173                         # Some filesystems such as CIFS return
 174                         # ENOENT which means st_nlink == 0.
 175                         return 0
 176                 raise
 177
 178 def unlockfile(mytuple):
 179         import fcntl
 180
 181         #XXX: Compatability hack.
 182         if len(mytuple) == 3:
 183                 lockfilename,myfd,unlinkfile = mytuple
 184                 locking_method = fcntl.flock
 185         elif len(mytuple) == 4:
 186                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 187         else:
 188                 raise InvalidData
 189
 190         if(myfd == HARDLINK_FD):
 191                 unhardlink_lockfile(lockfilename)
 192                 return True
 193
 194         # myfd may be None here due to myfd = mypath in lockfile()
 195         if isinstance(lockfilename, basestring) and \
 196                 not os.path.exists(lockfilename):
 197                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 198                 if myfd is not None:
 199                         os.close(myfd)
 200                 return False
 201
 202         try:
 203                 if myfd is None:
 204                         myfd = os.open(lockfilename, os.O_WRONLY,0660)
 205                         unlinkfile = 1
 206                 locking_method(myfd,fcntl.LOCK_UN)
 207         except OSError:
 208                 if isinstance(lockfilename, basestring):
 209                         os.close(myfd)
 210                 raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 211
 212         try:
 213                 # This sleep call was added to allow other processes that are
 214                 # waiting for a lock to be able to grab it before it is deleted.
 215                 # lockfile() already accounts for this situation, however, and
 216                 # the sleep here adds more time than is saved overall, so am
 217                 # commenting until it is proved necessary.
 218                 #time.sleep(0.0001)
 219                 if unlinkfile:
 220                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 221                         # We won the lock, so there isn't competition for it.
 222                         # We can safely delete the file.
 223                         writemsg(_("Got the lockfile...\n"), 1)
 224                         if _fstat_nlink(myfd) == 1:
 225                                 os.unlink(lockfilename)
 226                                 writemsg(_("Unlinked lockfile...\n"), 1)
 227                                 locking_method(myfd,fcntl.LOCK_UN)
 228                         else:
 229                                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 230                                 os.close(myfd)
 231                                 return False
 232         except Exception as e:
 233                 writemsg(_("Failed to get lock... someone took it.\n"), 1)
 234                 writemsg(str(e)+"\n",1)
 235
 236         # why test lockfilename?  because we may have been handed an
 237         # fd originally, and the caller might not like having their
 238         # open fd closed automatically on them.
 239         if isinstance(lockfilename, basestring):
 240                 os.close(myfd)
 241
 242         return True
 243
 244
 245
 246
 247 def hardlock_name(path):
 248         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 249
 250 def hardlink_is_mine(link,lock):
 251         try:
 252                 return os.stat(link).st_nlink == 2
 253         except OSError:
 254                 return False
 255
 256 def hardlink_lockfile(lockfilename, max_wait=14400):
 257         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 258         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 259         Then we HARDLINK the real lockfile to that private file.
 260         If our file can 2 references, then we have the lock. :)
 261         Otherwise we lather, rise, and repeat.
 262         We default to a 4 hour timeout.
 263         """
 264
 265         start_time = time.time()
 266         myhardlock = hardlock_name(lockfilename)
 267         reported_waiting = False
 268
 269         while(time.time() < (start_time + max_wait)):
 270                 # We only need it to exist.
 271                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0660)
 272                 os.close(myfd)
 273
 274                 if not os.path.exists(myhardlock):
 275                         raise FileNotFound(
 276                                 _("Created lockfile is missing: %(filename)s") % \
 277                                 {"filename" : myhardlock})
 278
 279                 try:
 280                         res = os.link(myhardlock, lockfilename)
 281                 except OSError:
 282                         pass
 283
 284                 if hardlink_is_mine(myhardlock, lockfilename):
 285                         # We have the lock.
 286                         if reported_waiting:
 287                                 writemsg("\n", noiselevel=-1)
 288                         return True
 289
 290                 if reported_waiting:
 291                         writemsg(".", noiselevel=-1)
 292                 else:
 293                         reported_waiting = True
 294                         from portage.const import PORTAGE_BIN_PATH
 295                         msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
 296                                 "%(bin_path)s/clean_locks can fix stuck locks.\n"
 297                                 "Lockfile: %(lockfilename)s\n") % \
 298                                 {"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
 299                         writemsg(msg, noiselevel=-1)
 300                 time.sleep(3)
 301
 302         os.unlink(myhardlock)
 303         return False
 304
 305 def unhardlink_lockfile(lockfilename):
 306         myhardlock = hardlock_name(lockfilename)
 307         if hardlink_is_mine(myhardlock, lockfilename):
 308                 # Make sure not to touch lockfilename unless we really have a lock.
 309                 try:
 310                         os.unlink(lockfilename)
 311                 except OSError:
 312                         pass
 313         try:
 314                 os.unlink(myhardlock)
 315         except OSError:
 316                 pass
 317
 318 def hardlock_cleanup(path, remove_all_locks=False):
 319         mypid  = str(os.getpid())
 320         myhost = os.uname()[1]
 321         mydl = os.listdir(path)
 322
 323         results = []
 324         mycount = 0
 325
 326         mylist = {}
 327         for x in mydl:
 328                 if os.path.isfile(path+"/"+x):
 329                         parts = x.split(".hardlock-")
 330                         if len(parts) == 2:
 331                                 filename = parts[0]
 332                                 hostpid  = parts[1].split("-")
 333                                 host  = "-".join(hostpid[:-1])
 334                                 pid   = hostpid[-1]
 335
 336                                 if filename not in mylist:
 337                                         mylist[filename] = {}
 338                                 if host not in mylist[filename]:
 339                                         mylist[filename][host] = []
 340                                 mylist[filename][host].append(pid)
 341
 342                                 mycount += 1
 343
 344
 345         results.append(_("Found %(count)s locks") % {"count":mycount})
 346
 347         for x in mylist:
 348                 if myhost in mylist[x] or remove_all_locks:
 349                         mylockname = hardlock_name(path+"/"+x)
 350                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 351                            not os.path.exists(path+"/"+x) or \
 352                                  remove_all_locks:
 353                                 for y in mylist[x]:
 354                                         for z in mylist[x][y]:
 355                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 356                                                 if filename == mylockname:
 357                                                         continue
 358                                                 try:
 359                                                         # We're sweeping through, unlinking everyone's locks.
 360                                                         os.unlink(filename)
 361                                                         results.append(_("Unlinked: ") + filename)
 362                                                 except OSError:
 363                                                         pass
 364                                 try:
 365                                         os.unlink(path+"/"+x)
 366                                         results.append(_("Unlinked: ") + path+"/"+x)
 367                                         os.unlink(mylockname)
 368                                         results.append(_("Unlinked: ") + mylockname)
 369                                 except OSError:
 370                                         pass
 371                         else:
 372                                 try:
 373                                         os.unlink(mylockname)
 374                                         results.append(_("Unlinked: ") + mylockname)
 375                                 except OSError:
 376                                         pass
 377
 378         return results
 379