pym/_emerge/BinpkgFetcher.py

   1 # Copyright 1999-2011 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from _emerge.AsynchronousLock import AsynchronousLock
   5 from _emerge.SpawnProcess import SpawnProcess
   6 try:
   7         from urllib.parse import urlparse as urllib_parse_urlparse
   8 except ImportError:
   9         from urlparse import urlparse as urllib_parse_urlparse
  10 import stat
  11 import sys
  12 import portage
  13 from portage import os
  14 from portage.util._pty import _create_pty_or_pipe
  15
  16 if sys.hexversion >= 0x3000000:
  17         long = int
  18
  19 class BinpkgFetcher(SpawnProcess):
  20
  21         __slots__ = ("pkg", "pretend",
  22                 "locked", "pkg_path", "_lock_obj")
  23
  24         def __init__(self, **kwargs):
  25                 SpawnProcess.__init__(self, **kwargs)
  26                 pkg = self.pkg
  27                 self.pkg_path = pkg.root_config.trees["bintree"].getname(pkg.cpv)
  28
  29         def _start(self):
  30
  31                 if self.cancelled:
  32                         return
  33
  34                 pkg = self.pkg
  35                 pretend = self.pretend
  36                 bintree = pkg.root_config.trees["bintree"]
  37                 settings = bintree.settings
  38                 use_locks = "distlocks" in settings.features
  39                 pkg_path = self.pkg_path
  40
  41                 if not pretend:
  42                         portage.util.ensure_dirs(os.path.dirname(pkg_path))
  43                         if use_locks:
  44                                 self.lock()
  45                 exists = os.path.exists(pkg_path)
  46                 resume = exists and os.path.basename(pkg_path) in bintree.invalids
  47                 if not (pretend or resume):
  48                         # Remove existing file or broken symlink.
  49                         try:
  50                                 os.unlink(pkg_path)
  51                         except OSError:
  52                                 pass
  53
  54                 # urljoin doesn't work correctly with
  55                 # unrecognized protocols like sftp
  56                 if bintree._remote_has_index:
  57                         rel_uri = bintree._remotepkgs[pkg.cpv].get("PATH")
  58                         if not rel_uri:
  59                                 rel_uri = pkg.cpv + ".tbz2"
  60                         remote_base_uri = bintree._remotepkgs[pkg.cpv]["BASE_URI"]
  61                         uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
  62                 else:
  63                         uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
  64                                 "/" + pkg.pf + ".tbz2"
  65
  66                 if pretend:
  67                         portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
  68                         self._set_returncode((self.pid, os.EX_OK << 8))
  69                         self.wait()
  70                         return
  71
  72                 protocol = urllib_parse_urlparse(uri)[0]
  73                 fcmd_prefix = "FETCHCOMMAND"
  74                 if resume:
  75                         fcmd_prefix = "RESUMECOMMAND"
  76                 fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
  77                 if not fcmd:
  78                         fcmd = settings.get(fcmd_prefix)
  79
  80                 fcmd_vars = {
  81                         "DISTDIR" : os.path.dirname(pkg_path),
  82                         "URI"     : uri,
  83                         "FILE"    : os.path.basename(pkg_path)
  84                 }
  85
  86                 fetch_env = dict(settings.items())
  87                 fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
  88                         for x in portage.util.shlex_split(fcmd)]
  89
  90                 if self.fd_pipes is None:
  91                         self.fd_pipes = {}
  92                 fd_pipes = self.fd_pipes
  93
  94                 # Redirect all output to stdout since some fetchers like
  95                 # wget pollute stderr (if portage detects a problem then it
  96                 # can send it's own message to stderr).
  97                 fd_pipes.setdefault(0, sys.stdin.fileno())
  98                 fd_pipes.setdefault(1, sys.stdout.fileno())
  99                 fd_pipes.setdefault(2, sys.stdout.fileno())
 100
 101                 self.args = fetch_args
 102                 self.env = fetch_env
 103                 if settings.selinux_enabled():
 104                         self._selinux_type = settings["PORTAGE_FETCH_T"]
 105                 SpawnProcess._start(self)
 106
 107         def _pipe(self, fd_pipes):
 108                 """When appropriate, use a pty so that fetcher progress bars,
 109                 like wget has, will work properly."""
 110                 if self.background or not sys.stdout.isatty():
 111                         # When the output only goes to a log file,
 112                         # there's no point in creating a pty.
 113                         return os.pipe()
 114                 stdout_pipe = None
 115                 if not self.background:
 116                         stdout_pipe = fd_pipes.get(1)
 117                 got_pty, master_fd, slave_fd = \
 118                         _create_pty_or_pipe(copy_term_size=stdout_pipe)
 119                 return (master_fd, slave_fd)
 120
 121         def _set_returncode(self, wait_retval):
 122                 SpawnProcess._set_returncode(self, wait_retval)
 123                 if not self.pretend and self.returncode == os.EX_OK:
 124                         # If possible, update the mtime to match the remote package if
 125                         # the fetcher didn't already do it automatically.
 126                         bintree = self.pkg.root_config.trees["bintree"]
 127                         if bintree._remote_has_index:
 128                                 remote_mtime = bintree._remotepkgs[self.pkg.cpv].get("MTIME")
 129                                 if remote_mtime is not None:
 130                                         try:
 131                                                 remote_mtime = long(remote_mtime)
 132                                         except ValueError:
 133                                                 pass
 134                                         else:
 135                                                 try:
 136                                                         local_mtime = os.stat(self.pkg_path)[stat.ST_MTIME]
 137                                                 except OSError:
 138                                                         pass
 139                                                 else:
 140                                                         if remote_mtime != local_mtime:
 141                                                                 try:
 142                                                                         os.utime(self.pkg_path,
 143                                                                                 (remote_mtime, remote_mtime))
 144                                                                 except OSError:
 145                                                                         pass
 146
 147                 if self.locked:
 148                         self.unlock()
 149
 150         def lock(self):
 151                 """
 152                 This raises an AlreadyLocked exception if lock() is called
 153                 while a lock is already held. In order to avoid this, call
 154                 unlock() or check whether the "locked" attribute is True
 155                 or False before calling lock().
 156                 """
 157                 if self._lock_obj is not None:
 158                         raise self.AlreadyLocked((self._lock_obj,))
 159
 160                 async_lock = AsynchronousLock(path=self.pkg_path,
 161                         scheduler=self.scheduler)
 162                 async_lock.start()
 163
 164                 if async_lock.wait() != os.EX_OK:
 165                         # TODO: Use CompositeTask for better handling, like in EbuildPhase.
 166                         raise AssertionError("AsynchronousLock failed with returncode %s" \
 167                                 % (async_lock.returncode,))
 168
 169                 self._lock_obj = async_lock
 170                 self.locked = True
 171
 172         class AlreadyLocked(portage.exception.PortageException):
 173                 pass
 174
 175         def unlock(self):
 176                 if self._lock_obj is None:
 177                         return
 178                 self._lock_obj.unlock()
 179                 self._lock_obj = None
 180                 self.locked = False
 181