pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6
   7 import errno, os, stat, time, types
   8 from portage.exception import DirectoryNotFound, FileNotFound, \
   9         InvalidData, TryAgain
  10 from portage.data import portage_gid
  11 from portage.util import writemsg
  12 from portage.localization import _
  13
  14 HARDLINK_FD = -2
  15
  16 # Used by emerge in order to disable the "waiting for lock" message
  17 # so that it doesn't interfere with the status display.
  18 _quiet = False
  19
  20 def lockdir(mydir):
  21         return lockfile(mydir,wantnewlockfile=1)
  22 def unlockdir(mylock):
  23         return unlockfile(mylock)
  24
  25 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  26         waiting_msg=None, flags=0):
  27         """Creates all dirs upto, the given dir. Creates a lockfile
  28         for the given directory as the file: directoryname+'.portage_lockfile'."""
  29         import fcntl
  30
  31         if not mypath:
  32                 raise InvalidData("Empty path given")
  33
  34         if type(mypath) == types.StringType and mypath[-1] == '/':
  35                 mypath = mypath[:-1]
  36
  37         if type(mypath) == types.FileType:
  38                 mypath = mypath.fileno()
  39         if type(mypath) == types.IntType:
  40                 lockfilename    = mypath
  41                 wantnewlockfile = 0
  42                 unlinkfile      = 0
  43         elif wantnewlockfile:
  44                 base, tail = os.path.split(mypath)
  45                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  46                 del base, tail
  47                 unlinkfile   = 1
  48         else:
  49                 lockfilename = mypath
  50
  51         if type(mypath) == types.StringType:
  52                 if not os.path.exists(os.path.dirname(mypath)):
  53                         raise DirectoryNotFound(os.path.dirname(mypath))
  54                 if not os.path.exists(lockfilename):
  55                         old_mask=os.umask(000)
  56                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  57                         try:
  58                                 if os.stat(lockfilename).st_gid != portage_gid:
  59                                         os.chown(lockfilename,os.getuid(),portage_gid)
  60                         except OSError, e:
  61                                 if e[0] == 2: # No such file or directory
  62                                         return lockfile(mypath, wantnewlockfile=wantnewlockfile,
  63                                                 unlinkfile=unlinkfile, waiting_msg=waiting_msg,
  64                                                 flags=flags)
  65                                 else:
  66                                         writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n");
  67                         os.umask(old_mask)
  68                 else:
  69                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  70
  71         elif type(mypath) == types.IntType:
  72                 myfd = mypath
  73
  74         else:
  75                 raise ValueError("Unknown type passed in '%s': '%s'" % \
  76                         (type(mypath), mypath))
  77
  78         # try for a non-blocking lock, if it's held, throw a message
  79         # we're waiting on lockfile and use a blocking attempt.
  80         locking_method = fcntl.lockf
  81         try:
  82                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
  83         except IOError, e:
  84                 if "errno" not in dir(e):
  85                         raise
  86                 if e.errno in (errno.EACCES, errno.EAGAIN):
  87                         # resource temp unavailable; eg, someone beat us to the lock.
  88                         if flags & os.O_NONBLOCK:
  89                                 raise TryAgain(mypath)
  90
  91                         global _quiet
  92                         if _quiet:
  93                                 pass
  94                         elif waiting_msg is None:
  95                                 if isinstance(mypath, int):
  96                                         print "waiting for lock on fd %i" % myfd
  97                                 else:
  98                                         print "waiting for lock on %s" % lockfilename
  99                         elif waiting_msg:
 100                                 print waiting_msg
 101                         # try for the exclusive lock now.
 102                         fcntl.lockf(myfd,fcntl.LOCK_EX)
 103                 elif e.errno == errno.ENOLCK:
 104                         # We're not allowed to lock on this FS.
 105                         os.close(myfd)
 106                         link_success = False
 107                         if lockfilename == str(lockfilename):
 108                                 if wantnewlockfile:
 109                                         try:
 110                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
 111                                                         os.unlink(lockfilename)
 112                                         except OSError:
 113                                                 pass
 114                                         link_success = hardlink_lockfile(lockfilename)
 115                         if not link_success:
 116                                 raise
 117                         locking_method = None
 118                         myfd = HARDLINK_FD
 119                 else:
 120                         raise
 121
 122
 123         if type(lockfilename) == types.StringType and \
 124                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 125                 # The file was deleted on us... Keep trying to make one...
 126                 os.close(myfd)
 127                 writemsg("lockfile recurse\n",1)
 128                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 129                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 130                         waiting_msg=waiting_msg, flags=flags)
 131
 132         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 133         return (lockfilename,myfd,unlinkfile,locking_method)
 134
 135 def _fstat_nlink(fd):
 136         """
 137         @param fd: an open file descriptor
 138         @type fd: Integer
 139         @rtype: Integer
 140         @return: the current number of hardlinks to the file
 141         """
 142         try:
 143                 return os.fstat(fd).st_nlink
 144         except EnvironmentError, e:
 145                 if e.errno == errno.ENOENT:
 146                         # Some filesystems such as CIFS return
 147                         # ENOENT which means st_nlink == 0.
 148                         return 0
 149                 raise
 150
 151 def unlockfile(mytuple):
 152         import fcntl
 153
 154         #XXX: Compatability hack.
 155         if len(mytuple) == 3:
 156                 lockfilename,myfd,unlinkfile = mytuple
 157                 locking_method = fcntl.flock
 158         elif len(mytuple) == 4:
 159                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 160         else:
 161                 raise InvalidData
 162
 163         if(myfd == HARDLINK_FD):
 164                 unhardlink_lockfile(lockfilename)
 165                 return True
 166
 167         # myfd may be None here due to myfd = mypath in lockfile()
 168         if type(lockfilename) == types.StringType and not os.path.exists(lockfilename):
 169                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 170                 if myfd is not None:
 171                         os.close(myfd)
 172                 return False
 173
 174         try:
 175                 if myfd is None:
 176                         myfd = os.open(lockfilename, os.O_WRONLY,0660)
 177                         unlinkfile = 1
 178                 locking_method(myfd,fcntl.LOCK_UN)
 179         except OSError:
 180                 if type(lockfilename) == types.StringType:
 181                         os.close(myfd)
 182                 raise IOError("Failed to unlock file '%s'\n" % lockfilename)
 183
 184         try:
 185                 # This sleep call was added to allow other processes that are
 186                 # waiting for a lock to be able to grab it before it is deleted.
 187                 # lockfile() already accounts for this situation, however, and
 188                 # the sleep here adds more time than is saved overall, so am
 189                 # commenting until it is proved necessary.
 190                 #time.sleep(0.0001)
 191                 if unlinkfile:
 192                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 193                         # We won the lock, so there isn't competition for it.
 194                         # We can safely delete the file.
 195                         writemsg("Got the lockfile...\n",1)
 196                         if _fstat_nlink(myfd) == 1:
 197                                 os.unlink(lockfilename)
 198                                 writemsg("Unlinked lockfile...\n",1)
 199                                 locking_method(myfd,fcntl.LOCK_UN)
 200                         else:
 201                                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 202                                 os.close(myfd)
 203                                 return False
 204         except Exception, e:
 205                 writemsg("Failed to get lock... someone took it.\n",1)
 206                 writemsg(str(e)+"\n",1)
 207
 208         # why test lockfilename?  because we may have been handed an
 209         # fd originally, and the caller might not like having their
 210         # open fd closed automatically on them.
 211         if type(lockfilename) == types.StringType:
 212                 os.close(myfd)
 213
 214         return True
 215
 216
 217
 218
 219 def hardlock_name(path):
 220         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 221
 222 def hardlink_is_mine(link,lock):
 223         try:
 224                 return os.stat(link).st_nlink == 2
 225         except OSError:
 226                 return False
 227
 228 def hardlink_lockfile(lockfilename, max_wait=14400):
 229         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 230         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 231         Then we HARDLINK the real lockfile to that private file.
 232         If our file can 2 references, then we have the lock. :)
 233         Otherwise we lather, rise, and repeat.
 234         We default to a 4 hour timeout.
 235         """
 236
 237         start_time = time.time()
 238         myhardlock = hardlock_name(lockfilename)
 239         reported_waiting = False
 240
 241         while(time.time() < (start_time + max_wait)):
 242                 # We only need it to exist.
 243                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0660)
 244                 os.close(myfd)
 245
 246                 if not os.path.exists(myhardlock):
 247                         raise FileNotFound(
 248                                 _("Created lockfile is missing: %(filename)s") % \
 249                                 {"filename" : myhardlock})
 250
 251                 try:
 252                         res = os.link(myhardlock, lockfilename)
 253                 except OSError:
 254                         pass
 255
 256                 if hardlink_is_mine(myhardlock, lockfilename):
 257                         # We have the lock.
 258                         if reported_waiting:
 259                                 print
 260                         return True
 261
 262                 if reported_waiting:
 263                         writemsg(".")
 264                 else:
 265                         reported_waiting = True
 266                         from portage.const import PORTAGE_BIN_PATH
 267                         print
 268                         print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)"
 269                         print "This is a feature to prevent distfiles corruption."
 270                         print "%s/clean_locks can fix stuck locks." % PORTAGE_BIN_PATH
 271                         print "Lockfile: " + lockfilename
 272                 time.sleep(3)
 273
 274         os.unlink(myhardlock)
 275         return False
 276
 277 def unhardlink_lockfile(lockfilename):
 278         myhardlock = hardlock_name(lockfilename)
 279         if hardlink_is_mine(myhardlock, lockfilename):
 280                 # Make sure not to touch lockfilename unless we really have a lock.
 281                 try:
 282                         os.unlink(lockfilename)
 283                 except OSError:
 284                         pass
 285         try:
 286                 os.unlink(myhardlock)
 287         except OSError:
 288                 pass
 289
 290 def hardlock_cleanup(path, remove_all_locks=False):
 291         mypid  = str(os.getpid())
 292         myhost = os.uname()[1]
 293         mydl = os.listdir(path)
 294
 295         results = []
 296         mycount = 0
 297
 298         mylist = {}
 299         for x in mydl:
 300                 if os.path.isfile(path+"/"+x):
 301                         parts = x.split(".hardlock-")
 302                         if len(parts) == 2:
 303                                 filename = parts[0]
 304                                 hostpid  = parts[1].split("-")
 305                                 host  = "-".join(hostpid[:-1])
 306                                 pid   = hostpid[-1]
 307
 308                                 if filename not in mylist:
 309                                         mylist[filename] = {}
 310                                 if host not in mylist[filename]:
 311                                         mylist[filename][host] = []
 312                                 mylist[filename][host].append(pid)
 313
 314                                 mycount += 1
 315
 316
 317         results.append("Found %(count)s locks" % {"count":mycount})
 318
 319         for x in mylist:
 320                 if myhost in mylist[x] or remove_all_locks:
 321                         mylockname = hardlock_name(path+"/"+x)
 322                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 323                            not os.path.exists(path+"/"+x) or \
 324                                  remove_all_locks:
 325                                 for y in mylist[x]:
 326                                         for z in mylist[x][y]:
 327                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 328                                                 if filename == mylockname:
 329                                                         continue
 330                                                 try:
 331                                                         # We're sweeping through, unlinking everyone's locks.
 332                                                         os.unlink(filename)
 333                                                         results.append(_("Unlinked: ") + filename)
 334                                                 except OSError:
 335                                                         pass
 336                                 try:
 337                                         os.unlink(path+"/"+x)
 338                                         results.append(_("Unlinked: ") + path+"/"+x)
 339                                         os.unlink(mylockname)
 340                                         results.append(_("Unlinked: ") + mylockname)
 341                                 except OSError:
 342                                         pass
 343                         else:
 344                                 try:
 345                                         os.unlink(mylockname)
 346                                         results.append(_("Unlinked: ") + mylockname)
 347                                 except OSError:
 348                                         pass
 349
 350         return results
 351