pym/portage/locks.py

   1 # portage: Lock management code
   2 # Copyright 2004-2010 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
   6         "hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
   7         "unhardlink_lockfile", "hardlock_cleanup"]
   8
   9 import errno
  10 import fcntl
  11 import stat
  12 import sys
  13 import time
  14
  15 import portage
  16 from portage import os
  17 from portage.const import PORTAGE_BIN_PATH
  18 from portage.exception import DirectoryNotFound, FileNotFound, \
  19         InvalidData, TryAgain, OperationNotPermitted, PermissionDenied
  20 from portage.data import portage_gid
  21 from portage.util import writemsg
  22 from portage.localization import _
  23
  24 if sys.hexversion >= 0x3000000:
  25         basestring = str
  26
  27 HARDLINK_FD = -2
  28
  29 # Used by emerge in order to disable the "waiting for lock" message
  30 # so that it doesn't interfere with the status display.
  31 _quiet = False
  32
  33 def lockdir(mydir, flags=0):
  34         return lockfile(mydir, wantnewlockfile=1, flags=flags)
  35 def unlockdir(mylock):
  36         return unlockfile(mylock)
  37
  38 def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
  39         waiting_msg=None, flags=0):
  40         """
  41         If wantnewlockfile is True then this creates a lockfile in the parent
  42         directory as the file: '.' + basename + '.portage_lockfile'.
  43         """
  44
  45         if not mypath:
  46                 raise InvalidData(_("Empty path given"))
  47
  48         if isinstance(mypath, basestring) and mypath[-1] == '/':
  49                 mypath = mypath[:-1]
  50
  51         if hasattr(mypath, 'fileno'):
  52                 mypath = mypath.fileno()
  53         if isinstance(mypath, int):
  54                 lockfilename    = mypath
  55                 wantnewlockfile = 0
  56                 unlinkfile      = 0
  57         elif wantnewlockfile:
  58                 base, tail = os.path.split(mypath)
  59                 lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
  60                 del base, tail
  61                 unlinkfile   = 1
  62         else:
  63                 lockfilename = mypath
  64
  65         if isinstance(mypath, basestring):
  66                 if not os.path.exists(os.path.dirname(mypath)):
  67                         raise DirectoryNotFound(os.path.dirname(mypath))
  68                 preexisting = os.path.exists(lockfilename)
  69                 old_mask = os.umask(000)
  70                 try:
  71                         try:
  72                                 myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
  73                         except OSError as e:
  74                                 func_call = "open('%s')" % lockfilename
  75                                 if e.errno == OperationNotPermitted.errno:
  76                                         raise OperationNotPermitted(func_call)
  77                                 elif e.errno == PermissionDenied.errno:
  78                                         raise PermissionDenied(func_call)
  79                                 else:
  80                                         raise
  81
  82                         if not preexisting:
  83                                 try:
  84                                         if os.stat(lockfilename).st_gid != portage_gid:
  85                                                 os.chown(lockfilename, -1, portage_gid)
  86                                 except OSError as e:
  87                                         if e.errno in (errno.ENOENT, errno.ESTALE):
  88                                                 return lockfile(mypath,
  89                                                         wantnewlockfile=wantnewlockfile,
  90                                                         unlinkfile=unlinkfile, waiting_msg=waiting_msg,
  91                                                         flags=flags)
  92                                         else:
  93                                                 writemsg("%s: chown('%s', -1, %d)\n" % \
  94                                                         (e, lockfilename, portage_gid), noiselevel=-1)
  95                                                 writemsg(_("Cannot chown a lockfile: '%s'\n") % \
  96                                                         lockfilename, noiselevel=-1)
  97                                                 writemsg(_("Group IDs of current user: %s\n") % \
  98                                                         " ".join(str(n) for n in os.getgroups()),
  99                                                         noiselevel=-1)
 100                 finally:
 101                         os.umask(old_mask)
 102
 103         elif isinstance(mypath, int):
 104                 myfd = mypath
 105
 106         else:
 107                 raise ValueError(_("Unknown type passed in '%s': '%s'") % \
 108                         (type(mypath), mypath))
 109
 110         # try for a non-blocking lock, if it's held, throw a message
 111         # we're waiting on lockfile and use a blocking attempt.
 112         locking_method = fcntl.lockf
 113         try:
 114                 fcntl.lockf(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 115         except IOError as e:
 116                 if "errno" not in dir(e):
 117                         raise
 118                 if e.errno in (errno.EACCES, errno.EAGAIN):
 119                         # resource temp unavailable; eg, someone beat us to the lock.
 120                         if flags & os.O_NONBLOCK:
 121                                 os.close(myfd)
 122                                 raise TryAgain(mypath)
 123
 124                         global _quiet
 125                         if _quiet:
 126                                 out = None
 127                         else:
 128                                 out = portage.output.EOutput()
 129                         if waiting_msg is None:
 130                                 if isinstance(mypath, int):
 131                                         waiting_msg = _("waiting for lock on fd %i") % myfd
 132                                 else:
 133                                         waiting_msg = _("waiting for lock on %s\n") % lockfilename
 134                         if out is not None:
 135                                 out.ebegin(waiting_msg)
 136                         # try for the exclusive lock now.
 137                         try:
 138                                 fcntl.lockf(myfd, fcntl.LOCK_EX)
 139                         except EnvironmentError as e:
 140                                 if out is not None:
 141                                         out.eend(1, str(e))
 142                                 raise
 143                         if out is not None:
 144                                 out.eend(os.EX_OK)
 145                 elif e.errno == errno.ENOLCK:
 146                         # We're not allowed to lock on this FS.
 147                         os.close(myfd)
 148                         link_success = False
 149                         if lockfilename == str(lockfilename):
 150                                 if wantnewlockfile:
 151                                         try:
 152                                                 if os.stat(lockfilename)[stat.ST_NLINK] == 1:
 153                                                         os.unlink(lockfilename)
 154                                         except OSError:
 155                                                 pass
 156                                         link_success = hardlink_lockfile(lockfilename)
 157                         if not link_success:
 158                                 raise
 159                         locking_method = None
 160                         myfd = HARDLINK_FD
 161                 else:
 162                         raise
 163
 164
 165         if isinstance(lockfilename, basestring) and \
 166                 myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
 167                 # The file was deleted on us... Keep trying to make one...
 168                 os.close(myfd)
 169                 writemsg(_("lockfile recurse\n"), 1)
 170                 lockfilename, myfd, unlinkfile, locking_method = lockfile(
 171                         mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
 172                         waiting_msg=waiting_msg, flags=flags)
 173
 174         writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1)
 175         return (lockfilename,myfd,unlinkfile,locking_method)
 176
 177 def _fstat_nlink(fd):
 178         """
 179         @param fd: an open file descriptor
 180         @type fd: Integer
 181         @rtype: Integer
 182         @return: the current number of hardlinks to the file
 183         """
 184         try:
 185                 return os.fstat(fd).st_nlink
 186         except EnvironmentError as e:
 187                 if e.errno in (errno.ENOENT, errno.ESTALE):
 188                         # Some filesystems such as CIFS return
 189                         # ENOENT which means st_nlink == 0.
 190                         return 0
 191                 raise
 192
 193 def unlockfile(mytuple):
 194
 195         #XXX: Compatability hack.
 196         if len(mytuple) == 3:
 197                 lockfilename,myfd,unlinkfile = mytuple
 198                 locking_method = fcntl.flock
 199         elif len(mytuple) == 4:
 200                 lockfilename,myfd,unlinkfile,locking_method = mytuple
 201         else:
 202                 raise InvalidData
 203
 204         if(myfd == HARDLINK_FD):
 205                 unhardlink_lockfile(lockfilename)
 206                 return True
 207
 208         # myfd may be None here due to myfd = mypath in lockfile()
 209         if isinstance(lockfilename, basestring) and \
 210                 not os.path.exists(lockfilename):
 211                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1)
 212                 if myfd is not None:
 213                         os.close(myfd)
 214                 return False
 215
 216         try:
 217                 if myfd is None:
 218                         myfd = os.open(lockfilename, os.O_WRONLY,0o660)
 219                         unlinkfile = 1
 220                 locking_method(myfd,fcntl.LOCK_UN)
 221         except OSError:
 222                 if isinstance(lockfilename, basestring):
 223                         os.close(myfd)
 224                 raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
 225
 226         try:
 227                 # This sleep call was added to allow other processes that are
 228                 # waiting for a lock to be able to grab it before it is deleted.
 229                 # lockfile() already accounts for this situation, however, and
 230                 # the sleep here adds more time than is saved overall, so am
 231                 # commenting until it is proved necessary.
 232                 #time.sleep(0.0001)
 233                 if unlinkfile:
 234                         locking_method(myfd,fcntl.LOCK_EX|fcntl.LOCK_NB)
 235                         # We won the lock, so there isn't competition for it.
 236                         # We can safely delete the file.
 237                         writemsg(_("Got the lockfile...\n"), 1)
 238                         if _fstat_nlink(myfd) == 1:
 239                                 os.unlink(lockfilename)
 240                                 writemsg(_("Unlinked lockfile...\n"), 1)
 241                                 locking_method(myfd,fcntl.LOCK_UN)
 242                         else:
 243                                 writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
 244                                 os.close(myfd)
 245                                 return False
 246         except SystemExit:
 247                 raise
 248         except Exception as e:
 249                 writemsg(_("Failed to get lock... someone took it.\n"), 1)
 250                 writemsg(str(e)+"\n",1)
 251
 252         # why test lockfilename?  because we may have been handed an
 253         # fd originally, and the caller might not like having their
 254         # open fd closed automatically on them.
 255         if isinstance(lockfilename, basestring):
 256                 os.close(myfd)
 257
 258         return True
 259
 260
 261
 262
 263 def hardlock_name(path):
 264         return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid())
 265
 266 def hardlink_is_mine(link,lock):
 267         try:
 268                 return os.stat(link).st_nlink == 2
 269         except OSError:
 270                 return False
 271
 272 def hardlink_lockfile(lockfilename, max_wait=14400):
 273         """Does the NFS, hardlink shuffle to ensure locking on the disk.
 274         We create a PRIVATE lockfile, that is just a placeholder on the disk.
 275         Then we HARDLINK the real lockfile to that private file.
 276         If our file can 2 references, then we have the lock. :)
 277         Otherwise we lather, rise, and repeat.
 278         We default to a 4 hour timeout.
 279         """
 280
 281         start_time = time.time()
 282         myhardlock = hardlock_name(lockfilename)
 283         reported_waiting = False
 284
 285         while(time.time() < (start_time + max_wait)):
 286                 # We only need it to exist.
 287                 myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0o660)
 288                 os.close(myfd)
 289
 290                 if not os.path.exists(myhardlock):
 291                         raise FileNotFound(
 292                                 _("Created lockfile is missing: %(filename)s") % \
 293                                 {"filename" : myhardlock})
 294
 295                 try:
 296                         res = os.link(myhardlock, lockfilename)
 297                 except OSError:
 298                         pass
 299
 300                 if hardlink_is_mine(myhardlock, lockfilename):
 301                         # We have the lock.
 302                         if reported_waiting:
 303                                 writemsg("\n", noiselevel=-1)
 304                         return True
 305
 306                 if reported_waiting:
 307                         writemsg(".", noiselevel=-1)
 308                 else:
 309                         reported_waiting = True
 310                         msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
 311                                 "%(bin_path)s/clean_locks can fix stuck locks.\n"
 312                                 "Lockfile: %(lockfilename)s\n") % \
 313                                 {"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
 314                         writemsg(msg, noiselevel=-1)
 315                 time.sleep(3)
 316
 317         os.unlink(myhardlock)
 318         return False
 319
 320 def unhardlink_lockfile(lockfilename):
 321         myhardlock = hardlock_name(lockfilename)
 322         if hardlink_is_mine(myhardlock, lockfilename):
 323                 # Make sure not to touch lockfilename unless we really have a lock.
 324                 try:
 325                         os.unlink(lockfilename)
 326                 except OSError:
 327                         pass
 328         try:
 329                 os.unlink(myhardlock)
 330         except OSError:
 331                 pass
 332
 333 def hardlock_cleanup(path, remove_all_locks=False):
 334         mypid  = str(os.getpid())
 335         myhost = os.uname()[1]
 336         mydl = os.listdir(path)
 337
 338         results = []
 339         mycount = 0
 340
 341         mylist = {}
 342         for x in mydl:
 343                 if os.path.isfile(path+"/"+x):
 344                         parts = x.split(".hardlock-")
 345                         if len(parts) == 2:
 346                                 filename = parts[0]
 347                                 hostpid  = parts[1].split("-")
 348                                 host  = "-".join(hostpid[:-1])
 349                                 pid   = hostpid[-1]
 350
 351                                 if filename not in mylist:
 352                                         mylist[filename] = {}
 353                                 if host not in mylist[filename]:
 354                                         mylist[filename][host] = []
 355                                 mylist[filename][host].append(pid)
 356
 357                                 mycount += 1
 358
 359
 360         results.append(_("Found %(count)s locks") % {"count":mycount})
 361
 362         for x in mylist:
 363                 if myhost in mylist[x] or remove_all_locks:
 364                         mylockname = hardlock_name(path+"/"+x)
 365                         if hardlink_is_mine(mylockname, path+"/"+x) or \
 366                            not os.path.exists(path+"/"+x) or \
 367                                  remove_all_locks:
 368                                 for y in mylist[x]:
 369                                         for z in mylist[x][y]:
 370                                                 filename = path+"/"+x+".hardlock-"+y+"-"+z
 371                                                 if filename == mylockname:
 372                                                         continue
 373                                                 try:
 374                                                         # We're sweeping through, unlinking everyone's locks.
 375                                                         os.unlink(filename)
 376                                                         results.append(_("Unlinked: ") + filename)
 377                                                 except OSError:
 378                                                         pass
 379                                 try:
 380                                         os.unlink(path+"/"+x)
 381                                         results.append(_("Unlinked: ") + path+"/"+x)
 382                                         os.unlink(mylockname)
 383                                         results.append(_("Unlinked: ") + mylockname)
 384                                 except OSError:
 385                                         pass
 386                         else:
 387                                 try:
 388                                         os.unlink(mylockname)
 389                                         results.append(_("Unlinked: ") + mylockname)
 390                                 except OSError:
 391                                         pass
 392
 393         return results
 394