pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6
   7 import errno, os, stat, time, types
   8 from portage.exception import InvalidData, DirectoryNotFound, FileNotFound
   9 from portage.data import portage_gid
  10 from portage.util import writemsg
  11 from portage.localization import _
  12
  13 HARDLINK_FD = -2
  14
  15 def lockdir(mydir):
  16         return lockfile(mydir,wantnewlockfile=1)
  17 def unlockdir(mylock):
  18         return unlockfile(mylock)
  19
  20 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None):
  21         """Creates all dirs upto, the given dir. Creates a lockfile
  22         for the given directory as the file: directoryname+'.portage_lockfile'."""
  23         import fcntl
  24
  25         if not mypath:
  26                 raise InvalidData, "Empty path given"
  27
  28         if type(mypath) == types.StringType and mypath[-1] == '/':
  29                 mypath = mypath[:-1]
  30
  31         if type(mypath) == types.FileType:
  32                 mypath = mypath.fileno()
  33         if type(mypath) == types.IntType:
  34                 lockfilename    = mypath
  35                 wantnewlockfile = 0
  36                 unlinkfile      = 0
  37         elif wantnewlockfile:
  38                 base, tail = os.path.split(mypath)
  39                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  40                 del base, tail
  41                 unlinkfile   = 1
  42         else:
  43                 lockfilename = mypath
  44
  45         if type(mypath) == types.StringType:
  46                 if not os.path.exists(os.path.dirname(mypath)):
  47                         raise DirectoryNotFound, os.path.dirname(mypath)
  48                 if not os.path.exists(lockfilename):
  49                         old_mask=os.umask(000)
  50                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  51                         try:
  52                                 if os.stat(lockfilename).st_gid != portage_gid:
  53                                         os.chown(lockfilename,os.getuid(),portage_gid)
  54                         except OSError, e:
  55                                 if e[0] == 2: # No such file or directory
  56                                         return lockfile(mypath, wantnewlockfile=wantnewlockfile,
  57                                                 unlinkfile=unlinkfile, waiting_msg=waiting_msg)
  58                                 else:
  59                                         writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n");
  60                         os.umask(old_mask)
  61                 else:
  62                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  63
  64         elif type(mypath) == types.IntType:
  65                 myfd = mypath
  66
  67         else:
  68                 raise ValueError, "Unknown type passed in '%s': '%s'" % (type(mypath),mypath)
  69
  70         # try for a non-blocking lock, if it's held, throw a message
  71         # we're waiting on lockfile and use a blocking attempt.
  72         locking_method = fcntl.lockf
  73         try:
  74                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
  75         except IOError, e:
  76                 if "errno" not in dir(e):
  77                         raise
  78                 if e.errno == errno.EAGAIN:
  79                         # resource temp unavailable; eg, someone beat us to the lock.
  80                         if waiting_msg is None:
  81                                 if isinstance(mypath, int):
  82                                         print "waiting for lock on fd %i" % myfd
  83                                 else:
  84                                         print "waiting for lock on %s" % lockfilename
  85                         elif waiting_msg:
  86                                 print waiting_msg
  87                         # try for the exclusive lock now.
  88                         fcntl.lockf(myfd,fcntl.LOCK_EX)
  89                 elif e.errno == errno.ENOLCK:
  90                         # We're not allowed to lock on this FS.
  91                         os.close(myfd)
  92                         link_success = False
  93                         if lockfilename == str(lockfilename):
  94                                 if wantnewlockfile:
  95                                         try:
  96                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
  97                                                         os.unlink(lockfilename)
  98                                         except OSError:
  99                                                 pass
 100                                         link_success = hardlink_lockfile(lockfilename)
 101                         if not link_success:
 102                                 raise
 103                         locking_method = None
 104                         myfd = HARDLINK_FD
 105                 else:
 106                         raise
 107
 108
 109         if type(lockfilename) == types.StringType and \
 110                 myfd != HARDLINK_FD and os.fstat(myfd).st_nlink == 0:
 111                 # The file was deleted on us... Keep trying to make one...
 112                 os.close(myfd)
 113                 writemsg("lockfile recurse\n",1)
 114                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 115                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 116                         waiting_msg=waiting_msg)
 117
 118         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 119         return (lockfilename,myfd,unlinkfile,locking_method)
 120
 121 def unlockfile(mytuple):
 122         import fcntl
 123
 124         #XXX: Compatability hack.
 125         if len(mytuple) == 3:
 126                 lockfilename,myfd,unlinkfile = mytuple
 127                 locking_method = fcntl.flock
 128         elif len(mytuple) == 4:
 129                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 130         else:
 131                 raise InvalidData
 132
 133         if(myfd == HARDLINK_FD):
 134                 unhardlink_lockfile(lockfilename)
 135                 return True
 136
 137         # myfd may be None here due to myfd = mypath in lockfile()
 138         if type(lockfilename) == types.StringType and not os.path.exists(lockfilename):
 139                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 140                 if myfd is not None:
 141                         os.close(myfd)
 142                 return False
 143
 144         try:
 145                 if myfd is None:
 146                         myfd = os.open(lockfilename, os.O_WRONLY,0660)
 147                         unlinkfile = 1
 148                 locking_method(myfd,fcntl.LOCK_UN)
 149         except OSError:
 150                 if type(lockfilename) == types.StringType:
 151                         os.close(myfd)
 152                 raise IOError, "Failed to unlock file '%s'\n" % lockfilename
 153
 154         try:
 155                 # This sleep call was added to allow other processes that are
 156                 # waiting for a lock to be able to grab it before it is deleted.
 157                 # lockfile() already accounts for this situation, however, and
 158                 # the sleep here adds more time than is saved overall, so am
 159                 # commenting until it is proved necessary.
 160                 #time.sleep(0.0001)
 161                 if unlinkfile:
 162                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 163                         # We won the lock, so there isn't competition for it.
 164                         # We can safely delete the file.
 165                         writemsg("Got the lockfile...\n",1)
 166                         if os.fstat(myfd).st_nlink == 1:
 167                                 os.unlink(lockfilename)
 168                                 writemsg("Unlinked lockfile...\n",1)
 169                                 locking_method(myfd,fcntl.LOCK_UN)
 170                         else:
 171                                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 172                                 os.close(myfd)
 173                                 return False
 174         except Exception, e:
 175                 writemsg("Failed to get lock... someone took it.\n",1)
 176                 writemsg(str(e)+"\n",1)
 177
 178         # why test lockfilename?  because we may have been handed an
 179         # fd originally, and the caller might not like having their
 180         # open fd closed automatically on them.
 181         if type(lockfilename) == types.StringType:
 182                 os.close(myfd)
 183
 184         return True
 185
 186
 187
 188
 189 def hardlock_name(path):
 190         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 191
 192 def hardlink_is_mine(link,lock):
 193         try:
 194                 return os.stat(link).st_nlink == 2
 195         except OSError:
 196                 return False
 197
 198 def hardlink_lockfile(lockfilename, max_wait=14400):
 199         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 200         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 201         Then we HARDLINK the real lockfile to that private file.
 202         If our file can 2 references, then we have the lock. :)
 203         Otherwise we lather, rise, and repeat.
 204         We default to a 4 hour timeout.
 205         """
 206
 207         start_time = time.time()
 208         myhardlock = hardlock_name(lockfilename)
 209         reported_waiting = False
 210
 211         while(time.time() < (start_time + max_wait)):
 212                 # We only need it to exist.
 213                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0660)
 214                 os.close(myfd)
 215
 216                 if not os.path.exists(myhardlock):
 217                         raise FileNotFound, _("Created lockfile is missing: %(filename)s") % {"filename":myhardlock}
 218
 219                 try:
 220                         res = os.link(myhardlock, lockfilename)
 221                 except OSError:
 222                         pass
 223
 224                 if hardlink_is_mine(myhardlock, lockfilename):
 225                         # We have the lock.
 226                         if reported_waiting:
 227                                 print
 228                         return True
 229
 230                 if reported_waiting:
 231                         writemsg(".")
 232                 else:
 233                         reported_waiting = True
 234                         from portage.const import PORTAGE_BIN_PATH
 235                         print
 236                         print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)"
 237                         print "This is a feature to prevent distfiles corruption."
 238                         print "%s/clean_locks can fix stuck locks." % PORTAGE_BIN_PATH
 239                         print "Lockfile: " + lockfilename
 240                 time.sleep(3)
 241
 242         os.unlink(myhardlock)
 243         return False
 244
 245 def unhardlink_lockfile(lockfilename):
 246         myhardlock = hardlock_name(lockfilename)
 247         if hardlink_is_mine(myhardlock, lockfilename):
 248                 # Make sure not to touch lockfilename unless we really have a lock.
 249                 try:
 250                         os.unlink(lockfilename)
 251                 except OSError:
 252                         pass
 253         try:
 254                 os.unlink(myhardlock)
 255         except OSError:
 256                 pass
 257
 258 def hardlock_cleanup(path, remove_all_locks=False):
 259         mypid  = str(os.getpid())
 260         myhost = os.uname()[1]
 261         mydl = os.listdir(path)
 262
 263         results = []
 264         mycount = 0
 265
 266         mylist = {}
 267         for x in mydl:
 268                 if os.path.isfile(path+"/"+x):
 269                         parts = x.split(".hardlock-")
 270                         if len(parts) == 2:
 271                                 filename = parts[0]
 272                                 hostpid  = parts[1].split("-")
 273                                 host  = "-".join(hostpid[:-1])
 274                                 pid   = hostpid[-1]
 275
 276                                 if not mylist.has_key(filename):
 277                                         mylist[filename] = {}
 278                                 if not mylist[filename].has_key(host):
 279                                         mylist[filename][host] = []
 280                                 mylist[filename][host].append(pid)
 281
 282                                 mycount += 1
 283
 284
 285         results.append("Found %(count)s locks" % {"count":mycount})
 286
 287         for x in mylist.keys():
 288                 if mylist[x].has_key(myhost) or remove_all_locks:
 289                         mylockname = hardlock_name(path+"/"+x)
 290                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 291                            not os.path.exists(path+"/"+x) or \
 292                                  remove_all_locks:
 293                                 for y in mylist[x].keys():
 294                                         for z in mylist[x][y]:
 295                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 296                                                 if filename == mylockname:
 297                                                         continue
 298                                                 try:
 299                                                         # We're sweeping through, unlinking everyone's locks.
 300                                                         os.unlink(filename)
 301                                                         results.append(_("Unlinked: ") + filename)
 302                                                 except OSError:
 303                                                         pass
 304                                 try:
 305                                         os.unlink(path+"/"+x)
 306                                         results.append(_("Unlinked: ") + path+"/"+x)
 307                                         os.unlink(mylockname)
 308                                         results.append(_("Unlinked: ") + mylockname)
 309                                 except OSError:
 310                                         pass
 311                         else:
 312                                 try:
 313                                         os.unlink(mylockname)
 314                                         results.append(_("Unlinked: ") + mylockname)
 315                                 except OSError:
 316                                         pass
 317
 318         return results
 319