pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6
   7 import errno, os, stat, time, types
   8 from portage.exception import InvalidData, DirectoryNotFound, FileNotFound
   9 from portage.data import portage_gid
  10 from portage.util import writemsg
  11 from portage.localization import _
  12
  13 HARDLINK_FD = -2
  14
  15 def lockdir(mydir):
  16         return lockfile(mydir,wantnewlockfile=1)
  17 def unlockdir(mylock):
  18         return unlockfile(mylock)
  19
  20 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None):
  21         """Creates all dirs upto, the given dir. Creates a lockfile
  22         for the given directory as the file: directoryname+'.portage_lockfile'."""
  23         import fcntl
  24
  25         if not mypath:
  26                 raise InvalidData("Empty path given")
  27
  28         if type(mypath) == types.StringType and mypath[-1] == '/':
  29                 mypath = mypath[:-1]
  30
  31         if type(mypath) == types.FileType:
  32                 mypath = mypath.fileno()
  33         if type(mypath) == types.IntType:
  34                 lockfilename    = mypath
  35                 wantnewlockfile = 0
  36                 unlinkfile      = 0
  37         elif wantnewlockfile:
  38                 base, tail = os.path.split(mypath)
  39                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  40                 del base, tail
  41                 unlinkfile   = 1
  42         else:
  43                 lockfilename = mypath
  44
  45         if type(mypath) == types.StringType:
  46                 if not os.path.exists(os.path.dirname(mypath)):
  47                         raise DirectoryNotFound(os.path.dirname(mypath))
  48                 if not os.path.exists(lockfilename):
  49                         old_mask=os.umask(000)
  50                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  51                         try:
  52                                 if os.stat(lockfilename).st_gid != portage_gid:
  53                                         os.chown(lockfilename,os.getuid(),portage_gid)
  54                         except OSError, e:
  55                                 if e[0] == 2: # No such file or directory
  56                                         return lockfile(mypath, wantnewlockfile=wantnewlockfile,
  57                                                 unlinkfile=unlinkfile, waiting_msg=waiting_msg)
  58                                 else:
  59                                         writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n");
  60                         os.umask(old_mask)
  61                 else:
  62                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  63
  64         elif type(mypath) == types.IntType:
  65                 myfd = mypath
  66
  67         else:
  68                 raise ValueError("Unknown type passed in '%s': '%s'" % \
  69                         (type(mypath), mypath))
  70
  71         # try for a non-blocking lock, if it's held, throw a message
  72         # we're waiting on lockfile and use a blocking attempt.
  73         locking_method = fcntl.lockf
  74         try:
  75                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
  76         except IOError, e:
  77                 if "errno" not in dir(e):
  78                         raise
  79                 if e.errno in (errno.EACCES, errno.EAGAIN):
  80                         # resource temp unavailable; eg, someone beat us to the lock.
  81                         if waiting_msg is None:
  82                                 if isinstance(mypath, int):
  83                                         print "waiting for lock on fd %i" % myfd
  84                                 else:
  85                                         print "waiting for lock on %s" % lockfilename
  86                         elif waiting_msg:
  87                                 print waiting_msg
  88                         # try for the exclusive lock now.
  89                         fcntl.lockf(myfd,fcntl.LOCK_EX)
  90                 elif e.errno == errno.ENOLCK:
  91                         # We're not allowed to lock on this FS.
  92                         os.close(myfd)
  93                         link_success = False
  94                         if lockfilename == str(lockfilename):
  95                                 if wantnewlockfile:
  96                                         try:
  97                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
  98                                                         os.unlink(lockfilename)
  99                                         except OSError:
 100                                                 pass
 101                                         link_success = hardlink_lockfile(lockfilename)
 102                         if not link_success:
 103                                 raise
 104                         locking_method = None
 105                         myfd = HARDLINK_FD
 106                 else:
 107                         raise
 108
 109
 110         if type(lockfilename) == types.StringType and \
 111                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 112                 # The file was deleted on us... Keep trying to make one...
 113                 os.close(myfd)
 114                 writemsg("lockfile recurse\n",1)
 115                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 116                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 117                         waiting_msg=waiting_msg)
 118
 119         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 120         return (lockfilename,myfd,unlinkfile,locking_method)
 121
 122 def _fstat_nlink(fd):
 123         """
 124         @param fd: an open file descriptor
 125         @type fd: Integer
 126         @rtype: Integer
 127         @return: the current number of hardlinks to the file
 128         """
 129         try:
 130                 return os.fstat(fd).st_nlink
 131         except EnvironmentError, e:
 132                 if e.errno == errno.ENOENT:
 133                         # Some filesystems such as CIFS return
 134                         # ENOENT which means st_nlink == 0.
 135                         return 0
 136                 raise
 137
 138 def unlockfile(mytuple):
 139         import fcntl
 140
 141         #XXX: Compatability hack.
 142         if len(mytuple) == 3:
 143                 lockfilename,myfd,unlinkfile = mytuple
 144                 locking_method = fcntl.flock
 145         elif len(mytuple) == 4:
 146                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 147         else:
 148                 raise InvalidData
 149
 150         if(myfd == HARDLINK_FD):
 151                 unhardlink_lockfile(lockfilename)
 152                 return True
 153
 154         # myfd may be None here due to myfd = mypath in lockfile()
 155         if type(lockfilename) == types.StringType and not os.path.exists(lockfilename):
 156                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 157                 if myfd is not None:
 158                         os.close(myfd)
 159                 return False
 160
 161         try:
 162                 if myfd is None:
 163                         myfd = os.open(lockfilename, os.O_WRONLY,0660)
 164                         unlinkfile = 1
 165                 locking_method(myfd,fcntl.LOCK_UN)
 166         except OSError:
 167                 if type(lockfilename) == types.StringType:
 168                         os.close(myfd)
 169                 raise IOError("Failed to unlock file '%s'\n" % lockfilename)
 170
 171         try:
 172                 # This sleep call was added to allow other processes that are
 173                 # waiting for a lock to be able to grab it before it is deleted.
 174                 # lockfile() already accounts for this situation, however, and
 175                 # the sleep here adds more time than is saved overall, so am
 176                 # commenting until it is proved necessary.
 177                 #time.sleep(0.0001)
 178                 if unlinkfile:
 179                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 180                         # We won the lock, so there isn't competition for it.
 181                         # We can safely delete the file.
 182                         writemsg("Got the lockfile...\n",1)
 183                         if _fstat_nlink(myfd) == 1:
 184                                 os.unlink(lockfilename)
 185                                 writemsg("Unlinked lockfile...\n",1)
 186                                 locking_method(myfd,fcntl.LOCK_UN)
 187                         else:
 188                                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 189                                 os.close(myfd)
 190                                 return False
 191         except Exception, e:
 192                 writemsg("Failed to get lock... someone took it.\n",1)
 193                 writemsg(str(e)+"\n",1)
 194
 195         # why test lockfilename?  because we may have been handed an
 196         # fd originally, and the caller might not like having their
 197         # open fd closed automatically on them.
 198         if type(lockfilename) == types.StringType:
 199                 os.close(myfd)
 200
 201         return True
 202
 203
 204
 205
 206 def hardlock_name(path):
 207         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 208
 209 def hardlink_is_mine(link,lock):
 210         try:
 211                 return os.stat(link).st_nlink == 2
 212         except OSError:
 213                 return False
 214
 215 def hardlink_lockfile(lockfilename, max_wait=14400):
 216         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 217         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 218         Then we HARDLINK the real lockfile to that private file.
 219         If our file can 2 references, then we have the lock. :)
 220         Otherwise we lather, rise, and repeat.
 221         We default to a 4 hour timeout.
 222         """
 223
 224         start_time = time.time()
 225         myhardlock = hardlock_name(lockfilename)
 226         reported_waiting = False
 227
 228         while(time.time() < (start_time + max_wait)):
 229                 # We only need it to exist.
 230                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0660)
 231                 os.close(myfd)
 232
 233                 if not os.path.exists(myhardlock):
 234                         raise FileNotFound(
 235                                 _("Created lockfile is missing: %(filename)s") % \
 236                                 {"filename" : myhardlock})
 237
 238                 try:
 239                         res = os.link(myhardlock, lockfilename)
 240                 except OSError:
 241                         pass
 242
 243                 if hardlink_is_mine(myhardlock, lockfilename):
 244                         # We have the lock.
 245                         if reported_waiting:
 246                                 print
 247                         return True
 248
 249                 if reported_waiting:
 250                         writemsg(".")
 251                 else:
 252                         reported_waiting = True
 253                         from portage.const import PORTAGE_BIN_PATH
 254                         print
 255                         print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)"
 256                         print "This is a feature to prevent distfiles corruption."
 257                         print "%s/clean_locks can fix stuck locks." % PORTAGE_BIN_PATH
 258                         print "Lockfile: " + lockfilename
 259                 time.sleep(3)
 260
 261         os.unlink(myhardlock)
 262         return False
 263
 264 def unhardlink_lockfile(lockfilename):
 265         myhardlock = hardlock_name(lockfilename)
 266         if hardlink_is_mine(myhardlock, lockfilename):
 267                 # Make sure not to touch lockfilename unless we really have a lock.
 268                 try:
 269                         os.unlink(lockfilename)
 270                 except OSError:
 271                         pass
 272         try:
 273                 os.unlink(myhardlock)
 274         except OSError:
 275                 pass
 276
 277 def hardlock_cleanup(path, remove_all_locks=False):
 278         mypid  = str(os.getpid())
 279         myhost = os.uname()[1]
 280         mydl = os.listdir(path)
 281
 282         results = []
 283         mycount = 0
 284
 285         mylist = {}
 286         for x in mydl:
 287                 if os.path.isfile(path+"/"+x):
 288                         parts = x.split(".hardlock-")
 289                         if len(parts) == 2:
 290                                 filename = parts[0]
 291                                 hostpid  = parts[1].split("-")
 292                                 host  = "-".join(hostpid[:-1])
 293                                 pid   = hostpid[-1]
 294
 295                                 if filename not in mylist:
 296                                         mylist[filename] = {}
 297                                 if host not in mylist[filename]:
 298                                         mylist[filename][host] = []
 299                                 mylist[filename][host].append(pid)
 300
 301                                 mycount += 1
 302
 303
 304         results.append("Found %(count)s locks" % {"count":mycount})
 305
 306         for x in mylist:
 307                 if myhost in mylist[x] or remove_all_locks:
 308                         mylockname = hardlock_name(path+"/"+x)
 309                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 310                            not os.path.exists(path+"/"+x) or \
 311                                  remove_all_locks:
 312                                 for y in mylist[x]:
 313                                         for z in mylist[x][y]:
 314                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 315                                                 if filename == mylockname:
 316                                                         continue
 317                                                 try:
 318                                                         # We're sweeping through, unlinking everyone's locks.
 319                                                         os.unlink(filename)
 320                                                         results.append(_("Unlinked: ") + filename)
 321                                                 except OSError:
 322                                                         pass
 323                                 try:
 324                                         os.unlink(path+"/"+x)
 325                                         results.append(_("Unlinked: ") + path+"/"+x)
 326                                         os.unlink(mylockname)
 327                                         results.append(_("Unlinked: ") + mylockname)
 328                                 except OSError:
 329                                         pass
 330                         else:
 331                                 try:
 332                                         os.unlink(mylockname)
 333                                         results.append(_("Unlinked: ") + mylockname)
 334                                 except OSError:
 335                                         pass
 336
 337         return results
 338