pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   7         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   8         "unhardlink_lockfile", "hardlock_cleanup"]
   9
  10 import errno, os, stat, time
  11 from portage.exception import DirectoryNotFound, FileNotFound, \
  12         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  13 from portage.data import portage_gid
  14 from portage.output import EOutput
  15 from portage.util import writemsg
  16 from portage.localization import _
  17
  18 HARDLINK_FD = -2
  19
  20 # Used by emerge in order to disable the "waiting for lock" message
  21 # so that it doesn't interfere with the status display.
  22 _quiet = False
  23
  24 def lockdir(mydir):
  25         return lockfile(mydir,wantnewlockfile=1)
  26 def unlockdir(mylock):
  27         return unlockfile(mylock)
  28
  29 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  30         waiting_msg=None, flags=0):
  31         """Creates all dirs upto, the given dir. Creates a lockfile
  32         for the given directory as the file: directoryname+'.portage_lockfile'."""
  33         import fcntl
  34
  35         if not mypath:
  36                 raise InvalidData("Empty path given")
  37
  38         if isinstance(mypath, basestring) and mypath[-1] == '/':
  39                 mypath = mypath[:-1]
  40
  41         if hasattr(mypath, 'fileno'):
  42                 mypath = mypath.fileno()
  43         if isinstance(mypath, int):
  44                 lockfilename    = mypath
  45                 wantnewlockfile = 0
  46                 unlinkfile      = 0
  47         elif wantnewlockfile:
  48                 base, tail = os.path.split(mypath)
  49                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  50                 del base, tail
  51                 unlinkfile   = 1
  52         else:
  53                 lockfilename = mypath
  54
  55         if isinstance(mypath, basestring):
  56                 if not os.path.exists(os.path.dirname(mypath)):
  57                         raise DirectoryNotFound(os.path.dirname(mypath))
  58                 preexisting = os.path.exists(lockfilename)
  59                 old_mask = os.umask(000)
  60                 try:
  61                         try:
  62                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0660)
  63                         except OSError, e:
  64                                 func_call = "open('%s')" % lockfilename
  65                                 if e.errno == OperationNotPermitted.errno:
  66                                         raise OperationNotPermitted(func_call)
  67                                 elif e.errno == PermissionDenied.errno:
  68                                         raise PermissionDenied(func_call)
  69                                 else:
  70                                         raise
  71
  72                         if not preexisting:
  73                                 try:
  74                                         if os.stat(lockfilename).st_gid != portage_gid:
  75                                                 os.chown(lockfilename, -1, portage_gid)
  76                                 except OSError, e:
  77                                         if e.errno in (errno.ENOENT, errno.ESTALE):
  78                                                 return lockfile(mypath,
  79                                                         wantnewlockfile=wantnewlockfile,
  80                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
  81                                                         flags=flags)
  82                                         else:
  83                                                 writemsg("Cannot chown a lockfile. This could " + \
  84                                                         "cause inconvenience later.\n")
  85
  86                 finally:
  87                         os.umask(old_mask)
  88
  89         elif isinstance(mypath, int):
  90                 myfd = mypath
  91
  92         else:
  93                 raise ValueError("Unknown type passed in '%s': '%s'" % \
  94                         (type(mypath), mypath))
  95
  96         # try for a non-blocking lock, if it's held, throw a message
  97         # we're waiting on lockfile and use a blocking attempt.
  98         locking_method = fcntl.lockf
  99         try:
 100                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 101         except IOError, e:
 102                 if "errno" not in dir(e):
 103                         raise
 104                 if e.errno in (errno.EACCES, errno.EAGAIN):
 105                         # resource temp unavailable; eg, someone beat us to the lock.
 106                         if flags & os.O_NONBLOCK:
 107                                 raise TryAgain(mypath)
 108
 109                         global _quiet
 110                         out = EOutput()
 111                         out.quiet = _quiet
 112                         if waiting_msg is None:
 113                                 if isinstance(mypath, int):
 114                                         waiting_msg = "waiting for lock on fd %i" % myfd
 115                                 else:
 116                                         waiting_msg = "waiting for lock on %s\n" % lockfilename
 117                         out.ebegin(waiting_msg)
 118                         # try for the exclusive lock now.
 119                         try:
 120                                 fcntl.lockf(myfd, fcntl.LOCK_EX)
 121                         except EnvironmentError, e:
 122                                 out.eend(1, str(e))
 123                                 raise
 124                         out.eend(os.EX_OK)
 125                 elif e.errno == errno.ENOLCK:
 126                         # We're not allowed to lock on this FS.
 127                         os.close(myfd)
 128                         link_success = False
 129                         if lockfilename == str(lockfilename):
 130                                 if wantnewlockfile:
 131                                         try:
 132                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
 133                                                         os.unlink(lockfilename)
 134                                         except OSError:
 135                                                 pass
 136                                         link_success = hardlink_lockfile(lockfilename)
 137                         if not link_success:
 138                                 raise
 139                         locking_method = None
 140                         myfd = HARDLINK_FD
 141                 else:
 142                         raise
 143
 144
 145         if isinstance(lockfilename, basestring) and \
 146                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 147                 # The file was deleted on us... Keep trying to make one...
 148                 os.close(myfd)
 149                 writemsg("lockfile recurse\n",1)
 150                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 151                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 152                         waiting_msg=waiting_msg, flags=flags)
 153
 154         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 155         return (lockfilename,myfd,unlinkfile,locking_method)
 156
 157 def _fstat_nlink(fd):
 158         """
 159         @param fd: an open file descriptor
 160         @type fd: Integer
 161         @rtype: Integer
 162         @return: the current number of hardlinks to the file
 163         """
 164         try:
 165                 return os.fstat(fd).st_nlink
 166         except EnvironmentError, e:
 167                 if e.errno in (errno.ENOENT, errno.ESTALE):
 168                         # Some filesystems such as CIFS return
 169                         # ENOENT which means st_nlink == 0.
 170                         return 0
 171                 raise
 172
 173 def unlockfile(mytuple):
 174         import fcntl
 175
 176         #XXX: Compatability hack.
 177         if len(mytuple) == 3:
 178                 lockfilename,myfd,unlinkfile = mytuple
 179                 locking_method = fcntl.flock
 180         elif len(mytuple) == 4:
 181                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 182         else:
 183                 raise InvalidData
 184
 185         if(myfd == HARDLINK_FD):
 186                 unhardlink_lockfile(lockfilename)
 187                 return True
 188
 189         # myfd may be None here due to myfd = mypath in lockfile()
 190         if isinstance(lockfilename, basestring) and \
 191                 not os.path.exists(lockfilename):
 192                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 193                 if myfd is not None:
 194                         os.close(myfd)
 195                 return False
 196
 197         try:
 198                 if myfd is None:
 199                         myfd = os.open(lockfilename, os.O_WRONLY,0660)
 200                         unlinkfile = 1
 201                 locking_method(myfd,fcntl.LOCK_UN)
 202         except OSError:
 203                 if isinstance(lockfilename, basestring):
 204                         os.close(myfd)
 205                 raise IOError("Failed to unlock file '%s'\n" % lockfilename)
 206
 207         try:
 208                 # This sleep call was added to allow other processes that are
 209                 # waiting for a lock to be able to grab it before it is deleted.
 210                 # lockfile() already accounts for this situation, however, and
 211                 # the sleep here adds more time than is saved overall, so am
 212                 # commenting until it is proved necessary.
 213                 #time.sleep(0.0001)
 214                 if unlinkfile:
 215                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 216                         # We won the lock, so there isn't competition for it.
 217                         # We can safely delete the file.
 218                         writemsg("Got the lockfile...\n",1)
 219                         if _fstat_nlink(myfd) == 1:
 220                                 os.unlink(lockfilename)
 221                                 writemsg("Unlinked lockfile...\n",1)
 222                                 locking_method(myfd,fcntl.LOCK_UN)
 223                         else:
 224                                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 225                                 os.close(myfd)
 226                                 return False
 227         except Exception, e:
 228                 writemsg("Failed to get lock... someone took it.\n",1)
 229                 writemsg(str(e)+"\n",1)
 230
 231         # why test lockfilename?  because we may have been handed an
 232         # fd originally, and the caller might not like having their
 233         # open fd closed automatically on them.
 234         if isinstance(lockfilename, basestring):
 235                 os.close(myfd)
 236
 237         return True
 238
 239
 240
 241
 242 def hardlock_name(path):
 243         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 244
 245 def hardlink_is_mine(link,lock):
 246         try:
 247                 return os.stat(link).st_nlink == 2
 248         except OSError:
 249                 return False
 250
 251 def hardlink_lockfile(lockfilename, max_wait=14400):
 252         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 253         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 254         Then we HARDLINK the real lockfile to that private file.
 255         If our file can 2 references, then we have the lock. :)
 256         Otherwise we lather, rise, and repeat.
 257         We default to a 4 hour timeout.
 258         """
 259
 260         start_time = time.time()
 261         myhardlock = hardlock_name(lockfilename)
 262         reported_waiting = False
 263
 264         while(time.time() < (start_time + max_wait)):
 265                 # We only need it to exist.
 266                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0660)
 267                 os.close(myfd)
 268
 269                 if not os.path.exists(myhardlock):
 270                         raise FileNotFound(
 271                                 _("Created lockfile is missing: %(filename)s") % \
 272                                 {"filename" : myhardlock})
 273
 274                 try:
 275                         res = os.link(myhardlock, lockfilename)
 276                 except OSError:
 277                         pass
 278
 279                 if hardlink_is_mine(myhardlock, lockfilename):
 280                         # We have the lock.
 281                         if reported_waiting:
 282                                 writemsg("\n", noiselevel=-1)
 283                         return True
 284
 285                 if reported_waiting:
 286                         writemsg(".", noiselevel=-1)
 287                 else:
 288                         reported_waiting = True
 289                         from portage.const import PORTAGE_BIN_PATH
 290                         msg = "\nWaiting on (hardlink) lockfile:" + \
 291                                 " (one '.' per 3 seconds)\n" + \
 292                                 "%s/clean_locks can fix stuck locks.\n" % PORTAGE_BIN_PATH + \
 293                                 "Lockfile: %s\n" % lockfilename
 294                         writemsg(msg, noiselevel=-1)
 295                 time.sleep(3)
 296
 297         os.unlink(myhardlock)
 298         return False
 299
 300 def unhardlink_lockfile(lockfilename):
 301         myhardlock = hardlock_name(lockfilename)
 302         if hardlink_is_mine(myhardlock, lockfilename):
 303                 # Make sure not to touch lockfilename unless we really have a lock.
 304                 try:
 305                         os.unlink(lockfilename)
 306                 except OSError:
 307                         pass
 308         try:
 309                 os.unlink(myhardlock)
 310         except OSError:
 311                 pass
 312
 313 def hardlock_cleanup(path, remove_all_locks=False):
 314         mypid  = str(os.getpid())
 315         myhost = os.uname()[1]
 316         mydl = os.listdir(path)
 317
 318         results = []
 319         mycount = 0
 320
 321         mylist = {}
 322         for x in mydl:
 323                 if os.path.isfile(path+"/"+x):
 324                         parts = x.split(".hardlock-")
 325                         if len(parts) == 2:
 326                                 filename = parts[0]
 327                                 hostpid  = parts[1].split("-")
 328                                 host  = "-".join(hostpid[:-1])
 329                                 pid   = hostpid[-1]
 330
 331                                 if filename not in mylist:
 332                                         mylist[filename] = {}
 333                                 if host not in mylist[filename]:
 334                                         mylist[filename][host] = []
 335                                 mylist[filename][host].append(pid)
 336
 337                                 mycount += 1
 338
 339
 340         results.append("Found %(count)s locks" % {"count":mycount})
 341
 342         for x in mylist:
 343                 if myhost in mylist[x] or remove_all_locks:
 344                         mylockname = hardlock_name(path+"/"+x)
 345                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 346                            not os.path.exists(path+"/"+x) or \
 347                                  remove_all_locks:
 348                                 for y in mylist[x]:
 349                                         for z in mylist[x][y]:
 350                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 351                                                 if filename == mylockname:
 352                                                         continue
 353                                                 try:
 354                                                         # We're sweeping through, unlinking everyone's locks.
 355                                                         os.unlink(filename)
 356                                                         results.append(_("Unlinked: ") + filename)
 357                                                 except OSError:
 358                                                         pass
 359                                 try:
 360                                         os.unlink(path+"/"+x)
 361                                         results.append(_("Unlinked: ") + path+"/"+x)
 362                                         os.unlink(mylockname)
 363                                         results.append(_("Unlinked: ") + mylockname)
 364                                 except OSError:
 365                                         pass
 366                         else:
 367                                 try:
 368                                         os.unlink(mylockname)
 369                                         results.append(_("Unlinked: ") + mylockname)
 370                                 except OSError:
 371                                         pass
 372
 373         return results
 374