pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   7         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   8         "unhardlink_lockfile", "hardlock_cleanup"]
   9
  10 import errno
  11 import stat
  12 import sys
  13 import time
  14 from portage import os
  15 from portage.exception import DirectoryNotFound, FileNotFound, \
  16         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  17 from portage.data import portage_gid
  18 from portage.output import EOutput
  19 from portage.util import writemsg
  20 from portage.localization import _
  21
  22 if sys.hexversion >= 0x3000000:
  23         basestring = str
  24
  25 HARDLINK_FD = -2
  26
  27 # Used by emerge in order to disable the "waiting for lock" message
  28 # so that it doesn't interfere with the status display.
  29 _quiet = False
  30
  31 def lockdir(mydir):
  32         return lockfile(mydir,wantnewlockfile=1)
  33 def unlockdir(mylock):
  34         return unlockfile(mylock)
  35
  36 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  37         waiting_msg=None, flags=0):
  38         """
  39         If wantnewlockfile is True then this creates a lockfile in the parent
  40         directory as the file: '.' + basename + '.portage_lockfile'.
  41         """
  42         import fcntl
  43
  44         if not mypath:
  45                 raise InvalidData(_("Empty path given"))
  46
  47         if isinstance(mypath, basestring) and mypath[-1] == '/':
  48                 mypath = mypath[:-1]
  49
  50         if hasattr(mypath, 'fileno'):
  51                 mypath = mypath.fileno()
  52         if isinstance(mypath, int):
  53                 lockfilename    = mypath
  54                 wantnewlockfile = 0
  55                 unlinkfile      = 0
  56         elif wantnewlockfile:
  57                 base, tail = os.path.split(mypath)
  58                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  59                 del base, tail
  60                 unlinkfile   = 1
  61         else:
  62                 lockfilename = mypath
  63
  64         if isinstance(mypath, basestring):
  65                 if not os.path.exists(os.path.dirname(mypath)):
  66                         raise DirectoryNotFound(os.path.dirname(mypath))
  67                 preexisting = os.path.exists(lockfilename)
  68                 old_mask = os.umask(000)
  69                 try:
  70                         try:
  71                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
  72                         except OSError as e:
  73                                 func_call = "open('%s')" % lockfilename
  74                                 if e.errno == OperationNotPermitted.errno:
  75                                         raise OperationNotPermitted(func_call)
  76                                 elif e.errno == PermissionDenied.errno:
  77                                         raise PermissionDenied(func_call)
  78                                 else:
  79                                         raise
  80
  81                         if not preexisting:
  82                                 try:
  83                                         if os.stat(lockfilename).st_gid != portage_gid:
  84                                                 os.chown(lockfilename, -1, portage_gid)
  85                                 except OSError as e:
  86                                         if e.errno in (errno.ENOENT, errno.ESTALE):
  87                                                 return lockfile(mypath,
  88                                                         wantnewlockfile=wantnewlockfile,
  89                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
  90                                                         flags=flags)
  91                                         else:
  92                                                 writemsg(_("Cannot chown a lockfile. This could "
  93                                                         "cause inconvenience later.\n"))
  94
  95                 finally:
  96                         os.umask(old_mask)
  97
  98         elif isinstance(mypath, int):
  99                 myfd = mypath
 100
 101         else:
 102                 raise ValueError(_("Unknown type passed in '%s': '%s'") % \
 103                         (type(mypath), mypath))
 104
 105         # try for a non-blocking lock, if it's held, throw a message
 106         # we're waiting on lockfile and use a blocking attempt.
 107         locking_method = fcntl.lockf
 108         try:
 109                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 110         except IOError as e:
 111                 if "errno" not in dir(e):
 112                         raise
 113                 if e.errno in (errno.EACCES, errno.EAGAIN):
 114                         # resource temp unavailable; eg, someone beat us to the lock.
 115                         if flags & os.O_NONBLOCK:
 116                                 raise TryAgain(mypath)
 117
 118                         global _quiet
 119                         out = EOutput()
 120                         out.quiet = _quiet
 121                         if waiting_msg is None:
 122                                 if isinstance(mypath, int):
 123                                         waiting_msg = _("waiting for lock on fd %i") % myfd
 124                                 else:
 125                                         waiting_msg = _("waiting for lock on %s\n") % lockfilename
 126                         out.ebegin(waiting_msg)
 127                         # try for the exclusive lock now.
 128                         try:
 129                                 fcntl.lockf(myfd, fcntl.LOCK_EX)
 130                         except EnvironmentError as e:
 131                                 out.eend(1, str(e))
 132                                 raise
 133                         out.eend(os.EX_OK)
 134                 elif e.errno == errno.ENOLCK:
 135                         # We're not allowed to lock on this FS.
 136                         os.close(myfd)
 137                         link_success = False
 138                         if lockfilename == str(lockfilename):
 139                                 if wantnewlockfile:
 140                                         try:
 141                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
 142                                                         os.unlink(lockfilename)
 143                                         except OSError:
 144                                                 pass
 145                                         link_success = hardlink_lockfile(lockfilename)
 146                         if not link_success:
 147                                 raise
 148                         locking_method = None
 149                         myfd = HARDLINK_FD
 150                 else:
 151                         raise
 152
 153
 154         if isinstance(lockfilename, basestring) and \
 155                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 156                 # The file was deleted on us... Keep trying to make one...
 157                 os.close(myfd)
 158                 writemsg(_("lockfile recurse\n"), 1)
 159                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 160                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 161                         waiting_msg=waiting_msg, flags=flags)
 162
 163         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 164         return (lockfilename,myfd,unlinkfile,locking_method)
 165
 166 def _fstat_nlink(fd):
 167         """
 168         @param fd: an open file descriptor
 169         @type fd: Integer
 170         @rtype: Integer
 171         @return: the current number of hardlinks to the file
 172         """
 173         try:
 174                 return os.fstat(fd).st_nlink
 175         except EnvironmentError as e:
 176                 if e.errno in (errno.ENOENT, errno.ESTALE):
 177                         # Some filesystems such as CIFS return
 178                         # ENOENT which means st_nlink == 0.
 179                         return 0
 180                 raise
 181
 182 def unlockfile(mytuple):
 183         import fcntl
 184
 185         #XXX: Compatability hack.
 186         if len(mytuple) == 3:
 187                 lockfilename,myfd,unlinkfile = mytuple
 188                 locking_method = fcntl.flock
 189         elif len(mytuple) == 4:
 190                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 191         else:
 192                 raise InvalidData
 193
 194         if(myfd == HARDLINK_FD):
 195                 unhardlink_lockfile(lockfilename)
 196                 return True
 197
 198         # myfd may be None here due to myfd = mypath in lockfile()
 199         if isinstance(lockfilename, basestring) and \
 200                 not os.path.exists(lockfilename):
 201                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 202                 if myfd is not None:
 203                         os.close(myfd)
 204                 return False
 205
 206         try:
 207                 if myfd is None:
 208                         myfd = os.open(lockfilename, os.O_WRONLY,0o660)
 209                         unlinkfile = 1
 210                 locking_method(myfd,fcntl.LOCK_UN)
 211         except OSError:
 212                 if isinstance(lockfilename, basestring):
 213                         os.close(myfd)
 214                 raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 215
 216         try:
 217                 # This sleep call was added to allow other processes that are
 218                 # waiting for a lock to be able to grab it before it is deleted.
 219                 # lockfile() already accounts for this situation, however, and
 220                 # the sleep here adds more time than is saved overall, so am
 221                 # commenting until it is proved necessary.
 222                 #time.sleep(0.0001)
 223                 if unlinkfile:
 224                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 225                         # We won the lock, so there isn't competition for it.
 226                         # We can safely delete the file.
 227                         writemsg(_("Got the lockfile...\n"), 1)
 228                         if _fstat_nlink(myfd) == 1:
 229                                 os.unlink(lockfilename)
 230                                 writemsg(_("Unlinked lockfile...\n"), 1)
 231                                 locking_method(myfd,fcntl.LOCK_UN)
 232                         else:
 233                                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 234                                 os.close(myfd)
 235                                 return False
 236         except Exception as e:
 237                 writemsg(_("Failed to get lock... someone took it.\n"), 1)
 238                 writemsg(str(e)+"\n",1)
 239
 240         # why test lockfilename?  because we may have been handed an
 241         # fd originally, and the caller might not like having their
 242         # open fd closed automatically on them.
 243         if isinstance(lockfilename, basestring):
 244                 os.close(myfd)
 245
 246         return True
 247
 248
 249
 250
 251 def hardlock_name(path):
 252         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 253
 254 def hardlink_is_mine(link,lock):
 255         try:
 256                 return os.stat(link).st_nlink == 2
 257         except OSError:
 258                 return False
 259
 260 def hardlink_lockfile(lockfilename, max_wait=14400):
 261         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 262         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 263         Then we HARDLINK the real lockfile to that private file.
 264         If our file can 2 references, then we have the lock. :)
 265         Otherwise we lather, rise, and repeat.
 266         We default to a 4 hour timeout.
 267         """
 268
 269         start_time = time.time()
 270         myhardlock = hardlock_name(lockfilename)
 271         reported_waiting = False
 272
 273         while(time.time() < (start_time + max_wait)):
 274                 # We only need it to exist.
 275                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0o660)
 276                 os.close(myfd)
 277
 278                 if not os.path.exists(myhardlock):
 279                         raise FileNotFound(
 280                                 _("Created lockfile is missing: %(filename)s") % \
 281                                 {"filename" : myhardlock})
 282
 283                 try:
 284                         res = os.link(myhardlock, lockfilename)
 285                 except OSError:
 286                         pass
 287
 288                 if hardlink_is_mine(myhardlock, lockfilename):
 289                         # We have the lock.
 290                         if reported_waiting:
 291                                 writemsg("\n", noiselevel=-1)
 292                         return True
 293
 294                 if reported_waiting:
 295                         writemsg(".", noiselevel=-1)
 296                 else:
 297                         reported_waiting = True
 298                         from portage.const import PORTAGE_BIN_PATH
 299                         msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
 300                                 "%(bin_path)s/clean_locks can fix stuck locks.\n"
 301                                 "Lockfile: %(lockfilename)s\n") % \
 302                                 {"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
 303                         writemsg(msg, noiselevel=-1)
 304                 time.sleep(3)
 305
 306         os.unlink(myhardlock)
 307         return False
 308
 309 def unhardlink_lockfile(lockfilename):
 310         myhardlock = hardlock_name(lockfilename)
 311         if hardlink_is_mine(myhardlock, lockfilename):
 312                 # Make sure not to touch lockfilename unless we really have a lock.
 313                 try:
 314                         os.unlink(lockfilename)
 315                 except OSError:
 316                         pass
 317         try:
 318                 os.unlink(myhardlock)
 319         except OSError:
 320                 pass
 321
 322 def hardlock_cleanup(path, remove_all_locks=False):
 323         mypid  = str(os.getpid())
 324         myhost = os.uname()[1]
 325         mydl = os.listdir(path)
 326
 327         results = []
 328         mycount = 0
 329
 330         mylist = {}
 331         for x in mydl:
 332                 if os.path.isfile(path+"/"+x):
 333                         parts = x.split(".hardlock-")
 334                         if len(parts) == 2:
 335                                 filename = parts[0]
 336                                 hostpid  = parts[1].split("-")
 337                                 host  = "-".join(hostpid[:-1])
 338                                 pid   = hostpid[-1]
 339
 340                                 if filename not in mylist:
 341                                         mylist[filename] = {}
 342                                 if host not in mylist[filename]:
 343                                         mylist[filename][host] = []
 344                                 mylist[filename][host].append(pid)
 345
 346                                 mycount += 1
 347
 348
 349         results.append(_("Found %(count)s locks") % {"count":mycount})
 350
 351         for x in mylist:
 352                 if myhost in mylist[x] or remove_all_locks:
 353                         mylockname = hardlock_name(path+"/"+x)
 354                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 355                            not os.path.exists(path+"/"+x) or \
 356                                  remove_all_locks:
 357                                 for y in mylist[x]:
 358                                         for z in mylist[x][y]:
 359                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 360                                                 if filename == mylockname:
 361                                                         continue
 362                                                 try:
 363                                                         # We're sweeping through, unlinking everyone's locks.
 364                                                         os.unlink(filename)
 365                                                         results.append(_("Unlinked: ") + filename)
 366                                                 except OSError:
 367                                                         pass
 368                                 try:
 369                                         os.unlink(path+"/"+x)
 370                                         results.append(_("Unlinked: ") + path+"/"+x)
 371                                         os.unlink(mylockname)
 372                                         results.append(_("Unlinked: ") + mylockname)
 373                                 except OSError:
 374                                         pass
 375                         else:
 376                                 try:
 377                                         os.unlink(mylockname)
 378                                         results.append(_("Unlinked: ") + mylockname)
 379                                 except OSError:
 380                                         pass
 381
 382         return results
 383