pym/portage_locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id: /var/cvsroot/gentoo-src/portage/pym/portage_locks.py,v 1.18.2.2 2005/01/16 02:35:33 carpaski Exp $
   5
   6
   7 import errno
   8 import os
   9 import stat
  10 import string
  11 import time
  12 import types
  13 import portage_exception
  14 import portage_file
  15 import portage_util
  16 import portage_data
  17 from portage_exec import atexit_register
  18 from portage_localization import _
  19
  20 HARDLINK_FD = -2
  21
  22 hardlock_path_list = []
  23 def clean_my_hardlocks():
  24         for x in hardlock_path_list:
  25                 hardlock_cleanup(x)
  26 def add_hardlock_file_to_cleanup(path):
  27         mypath = portage_file.normpath(path)
  28         if os.path.isfile(mypath):
  29                 mypath = os.path.dirname(mypath)
  30         if os.path.isdir(mypath):
  31                 hardlock_path_list = mypath[:]
  32
  33 atexit_register(clean_my_hardlocks)
  34
  35 def lockdir(mydir):
  36         return lockfile(mydir,wantnewlockfile=1)
  37 def unlockdir(mylock):
  38         return unlockfile(mylock)
  39
  40 def lockfile(mypath,wantnewlockfile=0,unlinkfile=0):
  41         """Creates all dirs upto, the given dir. Creates a lockfile
  42         for the given directory as the file: directoryname+'.portage_lockfile'."""
  43         import fcntl
  44
  45         if not mypath:
  46                 raise portage_exception.InvalidData, "Empty path given"
  47
  48         if type(mypath) == types.StringType and mypath[-1] == '/':
  49                 mypath = mypath[:-1]
  50
  51         if type(mypath) == types.FileType:
  52                 mypath = mypath.fileno()
  53         if type(mypath) == types.IntType:
  54                 lockfilename    = mypath
  55                 wantnewlockfile = 0
  56                 unlinkfile      = 0
  57         elif wantnewlockfile:
  58                 lockfilename = mypath+".portage_lockfile"
  59                 unlinkfile   = 1
  60         else:
  61                 lockfilename = mypath
  62
  63         if type(mypath) == types.StringType:
  64                 if not os.path.exists(os.path.dirname(mypath)):
  65                         raise portage_exception.DirectoryNotFound, os.path.dirname(mypath)
  66                 if not os.path.exists(lockfilename):
  67                         old_mask=os.umask(000)
  68                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  69                         try:
  70                                 if os.stat(lockfilename).st_gid != portage_data.portage_gid:
  71                                         os.chown(lockfilename,os.getuid(),portage_data.portage_gid)
  72                         except SystemExit, e:
  73                                 raise
  74                         except OSError, e:
  75                                 if e[0] == 2: # No such file or directory
  76                                         return lockfile(mypath,wantnewlockfile,unlinkfile)
  77                                 else:
  78                                         portage_util.writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n");
  79                         os.umask(old_mask)
  80                 else:
  81                         myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR,0660)
  82
  83         elif type(mypath) == types.IntType:
  84                 myfd = mypath
  85
  86         else:
  87                 raise ValueError, "Unknown type passed in '%s': '%s'" % (type(mypath),mypath)
  88
  89         # try for a non-blocking lock, if it's held, throw a message
  90         # we're waiting on lockfile and use a blocking attempt.
  91         locking_method = fcntl.lockf
  92         try:
  93                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
  94         except IOError, e:
  95                 if "errno" not in dir(e):
  96                         raise
  97                 if e.errno == errno.EAGAIN:
  98                         # resource temp unavailable; eg, someone beat us to the lock.
  99                         if type(mypath) == types.IntType:
 100                                 print "waiting for lock on fd %i" % myfd
 101                         else:
 102                                 print "waiting for lock on %s" % lockfilename
 103                         # try for the exclusive lock now.
 104                         fcntl.lockf(myfd,fcntl.LOCK_EX)
 105                 elif e.errno == errno.ENOLCK:
 106                         # We're not allowed to lock on this FS.
 107                         os.close(myfd)
 108                         link_success = False
 109                         if lockfilename == str(lockfilename):
 110                                 if wantnewlockfile:
 111                                         try:
 112                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
 113                                                         os.unlink(lockfilename)
 114                                         except Exception, e:
 115                                                 pass
 116                                         link_success = hardlink_lockfile(lockfilename)
 117                         if not link_success:
 118                                 raise
 119                         locking_method = None
 120                         myfd = HARDLINK_FD
 121                 else:
 122                         raise
 123
 124
 125         if type(lockfilename) == types.StringType and os.fstat(myfd).st_nlink != 1:
 126                 # The file was deleted on us... Keep trying to make one...
 127                 os.close(myfd)
 128                 portage_util.writemsg("lockfile recurse\n",1)
 129                 lockfilename,myfd,unlinkfile,locking_method = lockfile(mypath,wantnewlockfile,unlinkfile)
 130
 131         portage_util.writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 132         return (lockfilename,myfd,unlinkfile,locking_method)
 133
 134 def unlockfile(mytuple):
 135         import fcntl
 136
 137         #XXX: Compatability hack.
 138         if len(mytuple) == 3:
 139                 lockfilename,myfd,unlinkfile = mytuple
 140                 locking_method = fcntl.flock
 141         elif len(mytuple) == 4:
 142                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 143         else:
 144                 raise
 145
 146         if(myfd == HARDLINK_FD):
 147                 unhardlink_lockfile(lockfilename)
 148                 return True
 149
 150         if type(lockfilename) == types.StringType and os.fstat(myfd).st_nlink != 1:
 151                 portage_util.writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 152                 os.close(myfd)
 153                 return False
 154
 155         try:
 156                 if myfd == None:
 157                         myfd = os.open(lockfilename, os.O_WRONLY,0660)
 158                         unlinkfile = 1
 159                 locking_method(myfd,fcntl.LOCK_UN)
 160         except SystemExit, e:
 161                 raise
 162         except Exception, e:
 163                 if type(lockfilename) == types.StringType:
 164                         os.close(myfd)
 165                 raise IOError, "Failed to unlock file '%s'\n" % lockfilename
 166
 167         try:
 168                 # This sleep call was added to allow other processes that are
 169                 # waiting for a lock to be able to grab it before it is deleted.
 170                 # lockfile() already accounts for this situation, however, and
 171                 # the sleep here adds more time than is saved overall, so am
 172                 # commenting until it is proved necessary.
 173                 #time.sleep(0.0001)
 174                 if unlinkfile:
 175                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 176                         # We won the lock, so there isn't competition for it.
 177                         # We can safely delete the file.
 178                         portage_util.writemsg("Got the lockfile...\n",1)
 179                         #portage_util.writemsg("Unlinking...\n")
 180                         if os.fstat(myfd).st_nlink == 1:
 181                                 os.unlink(lockfilename)
 182                                 portage_util.writemsg("Unlinked lockfile...\n",1)
 183                                 locking_method(myfd,fcntl.LOCK_UN)
 184                         else:
 185                                 portage_util.writemsg("lockfile does not exist '%s'\n" % lockfilename,1)
 186                                 os.close(myfd)
 187                                 return False
 188         except SystemExit, e:
 189                 raise
 190         except Exception, e:
 191                 # We really don't care... Someone else has the lock.
 192                 # So it is their problem now.
 193                 portage_util.writemsg("Failed to get lock... someone took it.\n",1)
 194                 portage_util.writemsg(str(e)+"\n",1)
 195
 196         # why test lockfilename?  because we may have been handed an
 197         # fd originally, and the caller might not like having their
 198         # open fd closed automatically on them.
 199         if type(lockfilename) == types.StringType:
 200                 os.close(myfd)
 201
 202         return True
 203
 204
 205
 206
 207 def hardlock_name(path):
 208         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 209
 210 def hardlink_active(lock):
 211         if not os.path.exists(lock):
 212                 return False
 213         # XXXXXXXXXXXXXXXXXXXXXXXXXX
 214
 215 def hardlink_is_mine(link,lock):
 216         try:
 217                 myhls = os.stat(link)
 218                 mylfs = os.stat(lock)
 219         except SystemExit, e:
 220                 raise
 221         except:
 222                 myhls = None
 223                 mylfs = None
 224
 225         if myhls:
 226                 if myhls[stat.ST_NLINK] == 2:
 227                         return True
 228                 if mylfs:
 229                         if mylfs[stat.ST_INO] == myhls[stat.ST_INO]:
 230                                 return True
 231         return False
 232
 233 def hardlink_lockfile(lockfilename, max_wait=14400):
 234         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 235         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 236         Then we HARDLINK the real lockfile to that private file.
 237         If our file can 2 references, then we have the lock. :)
 238         Otherwise we lather, rise, and repeat.
 239         We default to a 4 hour timeout.
 240         """
 241
 242         add_hardlock_file_to_cleanup(lockfilename)
 243
 244         start_time = time.time()
 245         myhardlock = hardlock_name(lockfilename)
 246         reported_waiting = False
 247
 248         while(time.time() < (start_time + max_wait)):
 249                 # We only need it to exist.
 250                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0660)
 251                 os.close(myfd)
 252
 253                 if not os.path.exists(myhardlock):
 254                         raise portage_exception.FileNotFound, _("Created lockfile is missing: %(filename)s") % {"filename":myhardlock}
 255
 256                 try:
 257                         res = os.link(myhardlock, lockfilename)
 258                 except SystemExit, e:
 259                         raise
 260                 except Exception, e:
 261                         #print "lockfile(): Hardlink: Link failed."
 262                         #print "Exception: ",e
 263                         pass
 264
 265                 if hardlink_is_mine(myhardlock, lockfilename):
 266                         # We have the lock.
 267                         if reported_waiting:
 268                                 print
 269                         return True
 270
 271                 if reported_waiting:
 272                         portage_util.writemsg(".")
 273                 else:
 274                         reported_waiting = True
 275                         print
 276                         print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)"
 277                         print "This is a feature to prevent distfiles corruption."
 278                         print "/usr/lib/portage/bin/clean_locks can fix stuck locks."
 279                         print "Lockfile: " + lockfilename
 280                 time.sleep(3)
 281
 282         os.unlink(myhardlock)
 283         return False
 284
 285 def unhardlink_lockfile(lockfilename):
 286         myhardlock = hardlock_name(lockfilename)
 287         try:
 288                 if os.path.exists(myhardlock):
 289                         os.unlink(myhardlock)
 290                 if os.path.exists(lockfilename):
 291                         os.unlink(lockfilename)
 292         except SystemExit, e:
 293                 raise
 294         except:
 295                 portage_util.writemsg("Something strange happened to our hardlink locks.\n")
 296
 297 def hardlock_cleanup(path, remove_all_locks=False):
 298         mypid  = str(os.getpid())
 299         myhost = os.uname()[1]
 300         mydl = os.listdir(path)
 301
 302         results = []
 303         mycount = 0
 304
 305         mylist = {}
 306         for x in mydl:
 307                 if os.path.isfile(path+"/"+x):
 308                         parts = string.split(x, ".hardlock-")
 309                         if len(parts) == 2:
 310                                 filename = parts[0]
 311                                 hostpid  = string.split(parts[1],"-")
 312                                 host  = string.join(hostpid[:-1], "-")
 313                                 pid   = hostpid[-1]
 314
 315                                 if not mylist.has_key(filename):
 316                                         mylist[filename] = {}
 317                                 if not mylist[filename].has_key(host):
 318                                         mylist[filename][host] = []
 319                                 mylist[filename][host].append(pid)
 320
 321                                 mycount += 1
 322
 323
 324         results.append("Found %(count)s locks" % {"count":mycount})
 325
 326         for x in mylist.keys():
 327                 if mylist[x].has_key(myhost) or remove_all_locks:
 328                         mylockname = hardlock_name(path+"/"+x)
 329                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 330                            not os.path.exists(path+"/"+x) or \
 331                                  remove_all_locks:
 332                                 for y in mylist[x].keys():
 333                                         for z in mylist[x][y]:
 334                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 335                                                 if filename == mylockname:
 336                                                         continue
 337                                                 try:
 338                                                         # We're sweeping through, unlinking everyone's locks.
 339                                                         os.unlink(filename)
 340                                                         results.append(_("Unlinked: ") + filename)
 341                                                 except SystemExit, e:
 342                                                         raise
 343                                                 except Exception,e:
 344                                                         pass
 345                                 try:
 346                                         os.unlink(path+"/"+x)
 347                                         results.append(_("Unlinked: ") + path+"/"+x)
 348                                         os.unlink(mylockname)
 349                                         results.append(_("Unlinked: ") + mylockname)
 350                                 except SystemExit, e:
 351                                         raise
 352                                 except Exception,e:
 353                                         pass
 354                         else:
 355                                 try:
 356                                         os.unlink(mylockname)
 357                                         results.append(_("Unlinked: ") + mylockname)
 358                                 except SystemExit, e:
 359                                         raise
 360                                 except Exception,e:
 361                                         pass
 362
 363         return results
 364