pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6
   7 import errno, os, stat, time, types
   8 from portage.exception import InvalidData, DirectoryNotFound, FileNotFound
   9 from portage.data import portage_gid
  10 from portage.util import writemsg
  11 from portage.localization import _
  12
  13 HARDLINK_FD = -2
  14
  15 def lockdir(mydir):
  16         return lockfile(mydir,wantnewlockfile=1)
  17 def unlockdir(mylock):
  18         return unlockfile(mylock)
  19
  20 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None):
  21         """Creates all dirs upto, the given dir. Creates a lockfile
  22         for the given directory as the file: directoryname+'.portage_lockfile'."""
  23         import fcntl
  24
  25         if not mypath:
  26                 raise InvalidData, "Empty path given"
  27
  28         if type(mypath) == types.StringType and mypath[-1] == '/':
  29                 mypath = mypath[:-1]
  30
  31         if type(mypath) == types.FileType:
  32                 mypath = mypath.fileno()
  33         if type(mypath) == types.IntType:
  34                 lockfilename    = mypath
  35                 wantnewlockfile = 0
  36                 unlinkfile      = 0
  37         elif wantnewlockfile:
  38                 base, tail = os.path.split(mypath)
  39                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  40                 del base, tail
  41                 unlinkfile   = 1
  42         else:
  43                 lockfilename = mypath
  44
  45         if type(mypath) == types.StringType:
  46                 if not os.path.exists(os.path.dirname(mypath)):
  47                         raise DirectoryNotFound, os.path.dirname(mypath)
  48                 if not os.path.exists(lockfilename):
  49                         old_mask=os.umask(000)
  50                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  51                         try:
  52                                 if os.stat(lockfilename).st_gid != portage_gid:
  53                                         os.chown(lockfilename,os.getuid(),portage_gid)
  54                         except OSError, e:
  55                                 if e[0] == 2: # No such file or directory
  56                                         return lockfile(mypath,wantnewlockfile,unlinkfile)
  57                                 else:
  58                                         writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n");
  59                         os.umask(old_mask)
  60                 else:
  61                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  62
  63         elif type(mypath) == types.IntType:
  64                 myfd = mypath
  65
  66         else:
  67                 raise ValueError, "Unknown type passed in '%s': '%s'" % (type(mypath),mypath)
  68
  69         # try for a non-blocking lock, if it's held, throw a message
  70         # we're waiting on lockfile and use a blocking attempt.
  71         locking_method = fcntl.lockf
  72         try:
  73                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
  74         except IOError, e:
  75                 if "errno" not in dir(e):
  76                         raise
  77                 if e.errno == errno.EAGAIN:
  78                         # resource temp unavailable; eg, someone beat us to the lock.
  79                         if waiting_msg is None:
  80                                 if isinstance(mypath, int):
  81                                         print "waiting for lock on fd %i" % myfd
  82                                 else:
  83                                         print "waiting for lock on %s" % lockfilename
  84                         elif waiting_msg:
  85                                 print waiting_msg
  86                         # try for the exclusive lock now.
  87                         fcntl.lockf(myfd,fcntl.LOCK_EX)
  88                 elif e.errno == errno.ENOLCK:
  89                         # We're not allowed to lock on this FS.
  90                         os.close(myfd)
  91                         link_success = False
  92                         if lockfilename == str(lockfilename):
  93                                 if wantnewlockfile:
  94                                         try:
  95                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
  96                                                         os.unlink(lockfilename)
  97                                         except OSError:
  98                                                 pass
  99                                         link_success = hardlink_lockfile(lockfilename)
 100                         if not link_success:
 101                                 raise
 102                         locking_method = None
 103                         myfd = HARDLINK_FD
 104                 else:
 105                         raise
 106
 107
 108         if type(lockfilename) == types.StringType and \
 109                 myfd != HARDLINK_FD and os.fstat(myfd).st_nlink == 0:
 110                 # The file was deleted on us... Keep trying to make one...
 111                 os.close(myfd)
 112                 writemsg("lockfile recurse\n",1)
 113                 lockfilename,myfd,unlinkfile,locking_method = lockfile(mypath,wantnewlockfile,unlinkfile)
 114
 115         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 116         return (lockfilename,myfd,unlinkfile,locking_method)
 117
 118 def unlockfile(mytuple):
 119         import fcntl
 120
 121         #XXX: Compatability hack.
 122         if len(mytuple) == 3:
 123                 lockfilename,myfd,unlinkfile = mytuple
 124                 locking_method = fcntl.flock
 125         elif len(mytuple) == 4:
 126                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 127         else:
 128                 raise InvalidData
 129
 130         if(myfd == HARDLINK_FD):
 131                 unhardlink_lockfile(lockfilename)
 132                 return True
 133
 134         # myfd may be None here due to myfd = mypath in lockfile()
 135         if type(lockfilename) == types.StringType and not os.path.exists(lockfilename):
 136                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 137                 if myfd is not None:
 138                         os.close(myfd)
 139                 return False
 140
 141         try:
 142                 if myfd is None:
 143                         myfd = os.open(lockfilename, os.O_WRONLY,0660)
 144                         unlinkfile = 1
 145                 locking_method(myfd,fcntl.LOCK_UN)
 146         except OSError:
 147                 if type(lockfilename) == types.StringType:
 148                         os.close(myfd)
 149                 raise IOError, "Failed to unlock file '%s'\n" % lockfilename
 150
 151         try:
 152                 # This sleep call was added to allow other processes that are
 153                 # waiting for a lock to be able to grab it before it is deleted.
 154                 # lockfile() already accounts for this situation, however, and
 155                 # the sleep here adds more time than is saved overall, so am
 156                 # commenting until it is proved necessary.
 157                 #time.sleep(0.0001)
 158                 if unlinkfile:
 159                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 160                         # We won the lock, so there isn't competition for it.
 161                         # We can safely delete the file.
 162                         writemsg("Got the lockfile...\n",1)
 163                         if os.fstat(myfd).st_nlink == 1:
 164                                 os.unlink(lockfilename)
 165                                 writemsg("Unlinked lockfile...\n",1)
 166                                 locking_method(myfd,fcntl.LOCK_UN)
 167                         else:
 168                                 writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 169                                 os.close(myfd)
 170                                 return False
 171         except Exception, e:
 172                 writemsg("Failed to get lock... someone took it.\n",1)
 173                 writemsg(str(e)+"\n",1)
 174
 175         # why test lockfilename?  because we may have been handed an
 176         # fd originally, and the caller might not like having their
 177         # open fd closed automatically on them.
 178         if type(lockfilename) == types.StringType:
 179                 os.close(myfd)
 180
 181         return True
 182
 183
 184
 185
 186 def hardlock_name(path):
 187         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 188
 189 def hardlink_is_mine(link,lock):
 190         try:
 191                 return os.stat(link).st_nlink == 2
 192         except OSError:
 193                 return False
 194
 195 def hardlink_lockfile(lockfilename, max_wait=14400):
 196         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 197         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 198         Then we HARDLINK the real lockfile to that private file.
 199         If our file can 2 references, then we have the lock. :)
 200         Otherwise we lather, rise, and repeat.
 201         We default to a 4 hour timeout.
 202         """
 203
 204         start_time = time.time()
 205         myhardlock = hardlock_name(lockfilename)
 206         reported_waiting = False
 207
 208         while(time.time() < (start_time + max_wait)):
 209                 # We only need it to exist.
 210                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0660)
 211                 os.close(myfd)
 212
 213                 if not os.path.exists(myhardlock):
 214                         raise FileNotFound, _("Created lockfile is missing: %(filename)s") % {"filename":myhardlock}
 215
 216                 try:
 217                         res = os.link(myhardlock, lockfilename)
 218                 except OSError:
 219                         pass
 220
 221                 if hardlink_is_mine(myhardlock, lockfilename):
 222                         # We have the lock.
 223                         if reported_waiting:
 224                                 print
 225                         return True
 226
 227                 if reported_waiting:
 228                         writemsg(".")
 229                 else:
 230                         reported_waiting = True
 231                         print
 232                         print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)"
 233                         print "This is a feature to prevent distfiles corruption."
 234                         print "/usr/lib/portage/bin/clean_locks can fix stuck locks."
 235                         print "Lockfile: " + lockfilename
 236                 time.sleep(3)
 237
 238         os.unlink(myhardlock)
 239         return False
 240
 241 def unhardlink_lockfile(lockfilename):
 242         myhardlock = hardlock_name(lockfilename)
 243         if hardlink_is_mine(myhardlock, lockfilename):
 244                 # Make sure not to touch lockfilename unless we really have a lock.
 245                 try:
 246                         os.unlink(lockfilename)
 247                 except OSError:
 248                         pass
 249         try:
 250                 os.unlink(myhardlock)
 251         except OSError:
 252                 pass
 253
 254 def hardlock_cleanup(path, remove_all_locks=False):
 255         mypid  = str(os.getpid())
 256         myhost = os.uname()[1]
 257         mydl = os.listdir(path)
 258
 259         results = []
 260         mycount = 0
 261
 262         mylist = {}
 263         for x in mydl:
 264                 if os.path.isfile(path+"/"+x):
 265                         parts = x.split(".hardlock-")
 266                         if len(parts) == 2:
 267                                 filename = parts[0]
 268                                 hostpid  = parts[1].split("-")
 269                                 host  = "-".join(hostpid[:-1])
 270                                 pid   = hostpid[-1]
 271
 272                                 if not mylist.has_key(filename):
 273                                         mylist[filename] = {}
 274                                 if not mylist[filename].has_key(host):
 275                                         mylist[filename][host] = []
 276                                 mylist[filename][host].append(pid)
 277
 278                                 mycount += 1
 279
 280
 281         results.append("Found %(count)s locks" % {"count":mycount})
 282
 283         for x in mylist.keys():
 284                 if mylist[x].has_key(myhost) or remove_all_locks:
 285                         mylockname = hardlock_name(path+"/"+x)
 286                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 287                            not os.path.exists(path+"/"+x) or \
 288                                  remove_all_locks:
 289                                 for y in mylist[x].keys():
 290                                         for z in mylist[x][y]:
 291                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 292                                                 if filename == mylockname:
 293                                                         continue
 294                                                 try:
 295                                                         # We're sweeping through, unlinking everyone's locks.
 296                                                         os.unlink(filename)
 297                                                         results.append(_("Unlinked: ") + filename)
 298                                                 except OSError:
 299                                                         pass
 300                                 try:
 301                                         os.unlink(path+"/"+x)
 302                                         results.append(_("Unlinked: ") + path+"/"+x)
 303                                         os.unlink(mylockname)
 304                                         results.append(_("Unlinked: ") + mylockname)
 305                                 except OSError:
 306                                         pass
 307                         else:
 308                                 try:
 309                                         os.unlink(mylockname)
 310                                         results.append(_("Unlinked: ") + mylockname)
 311                                 except OSError:
 312                                         pass
 313
 314         return results
 315