pym/portage/package/ebuild/fetch.py

   1 # Copyright 2010 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import print_function
   5
   6 __all__ = ['fetch']
   7
   8 import codecs
   9 import errno
  10 import logging
  11 import random
  12 import re
  13 import shutil
  14 import stat
  15 import sys
  16 import tempfile
  17
  18
  19 import portage
  20 portage.proxy.lazyimport.lazyimport(globals(),
  21         'portage.package.ebuild.config:check_config_instance,config',
  22         'portage.package.ebuild.doebuild:doebuild_environment,' + \
  23                 '_doebuild_spawn',
  24         'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
  25 )
  26
  27 from portage import OrderedDict, os, selinux, _encodings, \
  28         _shell_quote, _unicode_encode
  29 from portage.checksum import perform_md5, verify_all
  30 from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
  31         GLOBAL_CONFIG_PATH
  32 from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
  33 from portage.exception import FileNotFound, OperationNotPermitted, \
  34         PermissionDenied, PortageException, TryAgain
  35 from portage.localization import _
  36 from portage.locks import lockfile, unlockfile
  37 from portage.manifest import Manifest
  38 from portage.output import colorize, EOutput
  39 from portage.util import apply_recursive_permissions, \
  40         apply_secpass_permissions, ensure_dirs, grabdict, shlex_split, \
  41         varexpand, writemsg, writemsg_level, writemsg_stdout
  42 from portage.process import spawn
  43
  44 _userpriv_spawn_kwargs = (
  45         ("uid",    portage_uid),
  46         ("gid",    portage_gid),
  47         ("groups", userpriv_groups),
  48         ("umask",  0o02),
  49 )
  50
  51 def _spawn_fetch(settings, args, **kwargs):
  52         """
  53         Spawn a process with appropriate settings for fetching, including
  54         userfetch and selinux support.
  55         """
  56
  57         global _userpriv_spawn_kwargs
  58
  59         # Redirect all output to stdout since some fetchers like
  60         # wget pollute stderr (if portage detects a problem then it
  61         # can send it's own message to stderr).
  62         if "fd_pipes" not in kwargs:
  63
  64                 kwargs["fd_pipes"] = {
  65                         0 : sys.stdin.fileno(),
  66                         1 : sys.stdout.fileno(),
  67                         2 : sys.stdout.fileno(),
  68                 }
  69
  70         if "userfetch" in settings.features and \
  71                 os.getuid() == 0 and portage_gid and portage_uid:
  72                 kwargs.update(_userpriv_spawn_kwargs)
  73
  74         spawn_func = spawn
  75
  76         if settings.selinux_enabled():
  77                 spawn_func = selinux.spawn_wrapper(spawn_func,
  78                         settings["PORTAGE_FETCH_T"])
  79
  80                 # bash is an allowed entrypoint, while most binaries are not
  81                 if args[0] != BASH_BINARY:
  82                         args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
  83
  84         # Ensure that EBUILD_PHASE is set to fetch, so that config.environ()
  85         # does not filter the calling environment (which may contain needed
  86         # proxy variables, as in bug #315421).
  87         phase_backup = settings.get('EBUILD_PHASE')
  88         settings['EBUILD_PHASE'] = 'fetch'
  89         try:
  90                 rval = spawn_func(args, env=settings.environ(), **kwargs)
  91         finally:
  92                 if phase_backup is None:
  93                         settings.pop('EBUILD_PHASE', None)
  94                 else:
  95                         settings['EBUILD_PHASE'] = phase_backup
  96
  97         return rval
  98
  99 _userpriv_test_write_file_cache = {}
 100 _userpriv_test_write_cmd_script = ">> %(file_path)s 2>/dev/null ; rval=$? ; " + \
 101         "rm -f  %(file_path)s ; exit $rval"
 102
 103 def _userpriv_test_write_file(settings, file_path):
 104         """
 105         Drop privileges and try to open a file for writing. The file may or
 106         may not exist, and the parent directory is assumed to exist. The file
 107         is removed before returning.
 108
 109         @param settings: A config instance which is passed to _spawn_fetch()
 110         @param file_path: A file path to open and write.
 111         @return: True if write succeeds, False otherwise.
 112         """
 113
 114         global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script
 115         rval = _userpriv_test_write_file_cache.get(file_path)
 116         if rval is not None:
 117                 return rval
 118
 119         args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \
 120                 {"file_path" : _shell_quote(file_path)}]
 121
 122         returncode = _spawn_fetch(settings, args)
 123
 124         rval = returncode == os.EX_OK
 125         _userpriv_test_write_file_cache[file_path] = rval
 126         return rval
 127
 128 def _checksum_failure_temp_file(distdir, basename):
 129         """
 130         First try to find a duplicate temp file with the same checksum and return
 131         that filename if available. Otherwise, use mkstemp to create a new unique
 132         filename._checksum_failure_.$RANDOM, rename the given file, and return the
 133         new filename. In any case, filename will be renamed or removed before this
 134         function returns a temp filename.
 135         """
 136
 137         filename = os.path.join(distdir, basename)
 138         size = os.stat(filename).st_size
 139         checksum = None
 140         tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*')
 141         for temp_filename in os.listdir(distdir):
 142                 if not tempfile_re.match(temp_filename):
 143                         continue
 144                 temp_filename = os.path.join(distdir, temp_filename)
 145                 try:
 146                         if size != os.stat(temp_filename).st_size:
 147                                 continue
 148                 except OSError:
 149                         continue
 150                 try:
 151                         temp_checksum = perform_md5(temp_filename)
 152                 except FileNotFound:
 153                         # Apparently the temp file disappeared. Let it go.
 154                         continue
 155                 if checksum is None:
 156                         checksum = perform_md5(filename)
 157                 if checksum == temp_checksum:
 158                         os.unlink(filename)
 159                         return temp_filename
 160
 161         fd, temp_filename = \
 162                 tempfile.mkstemp("", basename + "._checksum_failure_.", distdir)
 163         os.close(fd)
 164         os.rename(filename, temp_filename)
 165         return temp_filename
 166
 167 def _check_digests(filename, digests, show_errors=1):
 168         """
 169         Check digests and display a message if an error occurs.
 170         @return True if all digests match, False otherwise.
 171         """
 172         verified_ok, reason = verify_all(filename, digests)
 173         if not verified_ok:
 174                 if show_errors:
 175                         writemsg(_("!!! Previously fetched"
 176                                 " file: '%s'\n") % filename, noiselevel=-1)
 177                         writemsg(_("!!! Reason: %s\n") % reason[0],
 178                                 noiselevel=-1)
 179                         writemsg(_("!!! Got:      %s\n"
 180                                 "!!! Expected: %s\n") % \
 181                                 (reason[1], reason[2]), noiselevel=-1)
 182                 return False
 183         return True
 184
 185 def _check_distfile(filename, digests, eout, show_errors=1):
 186         """
 187         @return a tuple of (match, stat_obj) where match is True if filename
 188         matches all given digests (if any) and stat_obj is a stat result, or
 189         None if the file does not exist.
 190         """
 191         if digests is None:
 192                 digests = {}
 193         size = digests.get("size")
 194         if size is not None and len(digests) == 1:
 195                 digests = None
 196
 197         try:
 198                 st = os.stat(filename)
 199         except OSError:
 200                 return (False, None)
 201         if size is not None and size != st.st_size:
 202                 return (False, st)
 203         if not digests:
 204                 if size is not None:
 205                         eout.ebegin(_("%s size ;-)") % os.path.basename(filename))
 206                         eout.eend(0)
 207                 elif st.st_size == 0:
 208                         # Zero-byte distfiles are always invalid.
 209                         return (False, st)
 210         else:
 211                 if _check_digests(filename, digests, show_errors=show_errors):
 212                         eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
 213                                 " ".join(sorted(digests))))
 214                         eout.eend(0)
 215                 else:
 216                         return (False, st)
 217         return (True, st)
 218
 219 _fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
 220
 221 _size_suffix_map = {
 222         ''  : 0,
 223         'K' : 10,
 224         'M' : 20,
 225         'G' : 30,
 226         'T' : 40,
 227         'P' : 50,
 228         'E' : 60,
 229         'Z' : 70,
 230         'Y' : 80,
 231 }
 232
 233 def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
 234         "fetch files.  Will use digest file if available."
 235
 236         if not myuris:
 237                 return 1
 238
 239         features = mysettings.features
 240         restrict = mysettings.get("PORTAGE_RESTRICT","").split()
 241
 242         userfetch = secpass >= 2 and "userfetch" in features
 243         userpriv = secpass >= 2 and "userpriv" in features
 244
 245         # 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring.
 246         if "mirror" in restrict or \
 247            "nomirror" in restrict:
 248                 if ("mirror" in features) and ("lmirror" not in features):
 249                         # lmirror should allow you to bypass mirror restrictions.
 250                         # XXX: This is not a good thing, and is temporary at best.
 251                         print(_(">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch."))
 252                         return 1
 253
 254         # Generally, downloading the same file repeatedly from
 255         # every single available mirror is a waste of bandwidth
 256         # and time, so there needs to be a cap.
 257         checksum_failure_max_tries = 5
 258         v = checksum_failure_max_tries
 259         try:
 260                 v = int(mysettings.get("PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS",
 261                         checksum_failure_max_tries))
 262         except (ValueError, OverflowError):
 263                 writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
 264                         " contains non-integer value: '%s'\n") % \
 265                         mysettings["PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"], noiselevel=-1)
 266                 writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
 267                         "default value: %s\n") % checksum_failure_max_tries,
 268                         noiselevel=-1)
 269                 v = checksum_failure_max_tries
 270         if v < 1:
 271                 writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
 272                         " contains value less than 1: '%s'\n") % v, noiselevel=-1)
 273                 writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
 274                         "default value: %s\n") % checksum_failure_max_tries,
 275                         noiselevel=-1)
 276                 v = checksum_failure_max_tries
 277         checksum_failure_max_tries = v
 278         del v
 279
 280         fetch_resume_size_default = "350K"
 281         fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
 282         if fetch_resume_size is not None:
 283                 fetch_resume_size = "".join(fetch_resume_size.split())
 284                 if not fetch_resume_size:
 285                         # If it's undefined or empty, silently use the default.
 286                         fetch_resume_size = fetch_resume_size_default
 287                 match = _fetch_resume_size_re.match(fetch_resume_size)
 288                 if match is None or \
 289                         (match.group(2).upper() not in _size_suffix_map):
 290                         writemsg(_("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE"
 291                                 " contains an unrecognized format: '%s'\n") % \
 292                                 mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
 293                         writemsg(_("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE "
 294                                 "default value: %s\n") % fetch_resume_size_default,
 295                                 noiselevel=-1)
 296                         fetch_resume_size = None
 297         if fetch_resume_size is None:
 298                 fetch_resume_size = fetch_resume_size_default
 299                 match = _fetch_resume_size_re.match(fetch_resume_size)
 300         fetch_resume_size = int(match.group(1)) * \
 301                 2 ** _size_suffix_map[match.group(2).upper()]
 302
 303         # Behave like the package has RESTRICT="primaryuri" after a
 304         # couple of checksum failures, to increase the probablility
 305         # of success before checksum_failure_max_tries is reached.
 306         checksum_failure_primaryuri = 2
 307         thirdpartymirrors = mysettings.thirdpartymirrors()
 308
 309         # In the background parallel-fetch process, it's safe to skip checksum
 310         # verification of pre-existing files in $DISTDIR that have the correct
 311         # file size. The parent process will verify their checksums prior to
 312         # the unpack phase.
 313
 314         parallel_fetchonly = "PORTAGE_PARALLEL_FETCHONLY" in mysettings
 315         if parallel_fetchonly:
 316                 fetchonly = 1
 317
 318         check_config_instance(mysettings)
 319
 320         custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
 321                 CUSTOM_MIRRORS_FILE), recursive=1)
 322
 323         mymirrors=[]
 324
 325         if listonly or ("distlocks" not in features):
 326                 use_locks = 0
 327
 328         fetch_to_ro = 0
 329         if "skiprocheck" in features:
 330                 fetch_to_ro = 1
 331
 332         if not os.access(mysettings["DISTDIR"],os.W_OK) and fetch_to_ro:
 333                 if use_locks:
 334                         writemsg(colorize("BAD",
 335                                 _("!!! For fetching to a read-only filesystem, "
 336                                 "locking should be turned off.\n")), noiselevel=-1)
 337                         writemsg(_("!!! This can be done by adding -distlocks to "
 338                                 "FEATURES in /etc/make.conf\n"), noiselevel=-1)
 339 #                       use_locks = 0
 340
 341         # local mirrors are always added
 342         if "local" in custommirrors:
 343                 mymirrors += custommirrors["local"]
 344
 345         if "nomirror" in restrict or \
 346            "mirror" in restrict:
 347                 # We don't add any mirrors.
 348                 pass
 349         else:
 350                 if try_mirrors:
 351                         mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
 352
 353         skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
 354         pkgdir = mysettings.get("O")
 355         if not (pkgdir is None or skip_manifest):
 356                 mydigests = Manifest(
 357                         pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
 358         else:
 359                 # no digests because fetch was not called for a specific package
 360                 mydigests = {}
 361
 362         ro_distdirs = [x for x in \
 363                 shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
 364                 if os.path.isdir(x)]
 365
 366         fsmirrors = []
 367         for x in range(len(mymirrors)-1,-1,-1):
 368                 if mymirrors[x] and mymirrors[x][0]=='/':
 369                         fsmirrors += [mymirrors[x]]
 370                         del mymirrors[x]
 371
 372         restrict_fetch = "fetch" in restrict
 373         custom_local_mirrors = custommirrors.get("local", [])
 374         if restrict_fetch:
 375                 # With fetch restriction, a normal uri may only be fetched from
 376                 # custom local mirrors (if available).  A mirror:// uri may also
 377                 # be fetched from specific mirrors (effectively overriding fetch
 378                 # restriction, but only for specific mirrors).
 379                 locations = custom_local_mirrors
 380         else:
 381                 locations = mymirrors
 382
 383         file_uri_tuples = []
 384         # Check for 'items' attribute since OrderedDict is not a dict.
 385         if hasattr(myuris, 'items'):
 386                 for myfile, uri_set in myuris.items():
 387                         for myuri in uri_set:
 388                                 file_uri_tuples.append((myfile, myuri))
 389         else:
 390                 for myuri in myuris:
 391                         file_uri_tuples.append((os.path.basename(myuri), myuri))
 392
 393         filedict = OrderedDict()
 394         primaryuri_indexes={}
 395         primaryuri_dict = {}
 396         thirdpartymirror_uris = {}
 397         for myfile, myuri in file_uri_tuples:
 398                 if myfile not in filedict:
 399                         filedict[myfile]=[]
 400                         for y in range(0,len(locations)):
 401                                 filedict[myfile].append(locations[y]+"/distfiles/"+myfile)
 402                 if myuri[:9]=="mirror://":
 403                         eidx = myuri.find("/", 9)
 404                         if eidx != -1:
 405                                 mirrorname = myuri[9:eidx]
 406                                 path = myuri[eidx+1:]
 407
 408                                 # Try user-defined mirrors first
 409                                 if mirrorname in custommirrors:
 410                                         for cmirr in custommirrors[mirrorname]:
 411                                                 filedict[myfile].append(
 412                                                         cmirr.rstrip("/") + "/" + path)
 413
 414                                 # now try the official mirrors
 415                                 if mirrorname in thirdpartymirrors:
 416                                         random.shuffle(thirdpartymirrors[mirrorname])
 417
 418                                         uris = [locmirr.rstrip("/") + "/" + path \
 419                                                 for locmirr in thirdpartymirrors[mirrorname]]
 420                                         filedict[myfile].extend(uris)
 421                                         thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
 422
 423                                 if not filedict[myfile]:
 424                                         writemsg(_("No known mirror by the name: %s\n") % (mirrorname))
 425                         else:
 426                                 writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1)
 427                                 writemsg("  %s\n" % (myuri), noiselevel=-1)
 428                 else:
 429                         if restrict_fetch:
 430                                 # Only fetch from specific mirrors is allowed.
 431                                 continue
 432                         if "primaryuri" in restrict:
 433                                 # Use the source site first.
 434                                 if myfile in primaryuri_indexes:
 435                                         primaryuri_indexes[myfile] += 1
 436                                 else:
 437                                         primaryuri_indexes[myfile] = 0
 438                                 filedict[myfile].insert(primaryuri_indexes[myfile], myuri)
 439                         else:
 440                                 filedict[myfile].append(myuri)
 441                         primaryuris = primaryuri_dict.get(myfile)
 442                         if primaryuris is None:
 443                                 primaryuris = []
 444                                 primaryuri_dict[myfile] = primaryuris
 445                         primaryuris.append(myuri)
 446
 447         # Prefer thirdpartymirrors over normal mirrors in cases when
 448         # the file does not yet exist on the normal mirrors.
 449         for myfile, uris in thirdpartymirror_uris.items():
 450                 primaryuri_dict.setdefault(myfile, []).extend(uris)
 451
 452         can_fetch=True
 453
 454         if listonly:
 455                 can_fetch = False
 456
 457         if can_fetch and not fetch_to_ro:
 458                 global _userpriv_test_write_file_cache
 459                 dirmode  = 0o2070
 460                 filemode =   0o60
 461                 modemask =    0o2
 462                 dir_gid = portage_gid
 463                 if "FAKED_MODE" in mysettings:
 464                         # When inside fakeroot, directories with portage's gid appear
 465                         # to have root's gid. Therefore, use root's gid instead of
 466                         # portage's gid to avoid spurrious permissions adjustments
 467                         # when inside fakeroot.
 468                         dir_gid = 0
 469                 distdir_dirs = [""]
 470                 try:
 471
 472                         for x in distdir_dirs:
 473                                 mydir = os.path.join(mysettings["DISTDIR"], x)
 474                                 write_test_file = os.path.join(
 475                                         mydir, ".__portage_test_write__")
 476
 477                                 try:
 478                                         st = os.stat(mydir)
 479                                 except OSError:
 480                                         st = None
 481
 482                                 if st is not None and stat.S_ISDIR(st.st_mode):
 483                                         if not (userfetch or userpriv):
 484                                                 continue
 485                                         if _userpriv_test_write_file(mysettings, write_test_file):
 486                                                 continue
 487
 488                                 _userpriv_test_write_file_cache.pop(write_test_file, None)
 489                                 if ensure_dirs(mydir, gid=dir_gid, mode=dirmode, mask=modemask):
 490                                         if st is None:
 491                                                 # The directory has just been created
 492                                                 # and therefore it must be empty.
 493                                                 continue
 494                                         writemsg(_("Adjusting permissions recursively: '%s'\n") % mydir,
 495                                                 noiselevel=-1)
 496                                         def onerror(e):
 497                                                 raise # bail out on the first error that occurs during recursion
 498                                         if not apply_recursive_permissions(mydir,
 499                                                 gid=dir_gid, dirmode=dirmode, dirmask=modemask,
 500                                                 filemode=filemode, filemask=modemask, onerror=onerror):
 501                                                 raise OperationNotPermitted(
 502                                                         _("Failed to apply recursive permissions for the portage group."))
 503                 except PortageException as e:
 504                         if not os.path.isdir(mysettings["DISTDIR"]):
 505                                 writemsg("!!! %s\n" % str(e), noiselevel=-1)
 506                                 writemsg(_("!!! Directory Not Found: DISTDIR='%s'\n") % mysettings["DISTDIR"], noiselevel=-1)
 507                                 writemsg(_("!!! Fetching will fail!\n"), noiselevel=-1)
 508
 509         if can_fetch and \
 510                 not fetch_to_ro and \
 511                 not os.access(mysettings["DISTDIR"], os.W_OK):
 512                 writemsg(_("!!! No write access to '%s'\n") % mysettings["DISTDIR"],
 513                         noiselevel=-1)
 514                 can_fetch = False
 515
 516         distdir_writable = can_fetch and not fetch_to_ro
 517         failed_files = set()
 518         restrict_fetch_msg = False
 519
 520         for myfile in filedict:
 521                 """
 522                 fetched  status
 523                 0        nonexistent
 524                 1        partially downloaded
 525                 2        completely downloaded
 526                 """
 527                 fetched = 0
 528
 529                 orig_digests = mydigests.get(myfile, {})
 530                 size = orig_digests.get("size")
 531                 if size == 0:
 532                         # Zero-byte distfiles are always invalid, so discard their digests.
 533                         del mydigests[myfile]
 534                         orig_digests.clear()
 535                         size = None
 536                 pruned_digests = orig_digests
 537                 if parallel_fetchonly:
 538                         pruned_digests = {}
 539                         if size is not None:
 540                                 pruned_digests["size"] = size
 541
 542                 myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
 543                 has_space = True
 544                 has_space_superuser = True
 545                 file_lock = None
 546                 if listonly:
 547                         writemsg_stdout("\n", noiselevel=-1)
 548                 else:
 549                         # check if there is enough space in DISTDIR to completely store myfile
 550                         # overestimate the filesize so we aren't bitten by FS overhead
 551                         if size is not None and hasattr(os, "statvfs"):
 552                                 vfs_stat = os.statvfs(mysettings["DISTDIR"])
 553                                 try:
 554                                         mysize = os.stat(myfile_path).st_size
 555                                 except OSError as e:
 556                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 557                                                 raise
 558                                         del e
 559                                         mysize = 0
 560                                 if (size - mysize + vfs_stat.f_bsize) >= \
 561                                         (vfs_stat.f_bsize * vfs_stat.f_bavail):
 562
 563                                         if (size - mysize + vfs_stat.f_bsize) >= \
 564                                                 (vfs_stat.f_bsize * vfs_stat.f_bfree):
 565                                                 has_space_superuser = False
 566
 567                                         if not has_space_superuser:
 568                                                 has_space = False
 569                                         elif secpass < 2:
 570                                                 has_space = False
 571                                         elif userfetch:
 572                                                 has_space = False
 573
 574                         if not has_space:
 575                                 writemsg(_("!!! Insufficient space to store %s in %s\n") % \
 576                                         (myfile, mysettings["DISTDIR"]), noiselevel=-1)
 577
 578                                 if has_space_superuser:
 579                                         writemsg(_("!!! Insufficient privileges to use "
 580                                                 "remaining space.\n"), noiselevel=-1)
 581                                         if userfetch:
 582                                                 writemsg(_("!!! You may set FEATURES=\"-userfetch\""
 583                                                         " in /etc/make.conf in order to fetch with\n"
 584                                                         "!!! superuser privileges.\n"), noiselevel=-1)
 585
 586                         if distdir_writable and use_locks:
 587
 588                                 lock_kwargs = {}
 589                                 if fetchonly:
 590                                         lock_kwargs["flags"] = os.O_NONBLOCK
 591
 592                                 try:
 593                                         file_lock = lockfile(myfile_path,
 594                                                 wantnewlockfile=1, **lock_kwargs)
 595                                 except TryAgain:
 596                                         writemsg(_(">>> File '%s' is already locked by "
 597                                                 "another fetcher. Continuing...\n") % myfile,
 598                                                 noiselevel=-1)
 599                                         continue
 600                 try:
 601                         if not listonly:
 602
 603                                 eout = EOutput()
 604                                 eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
 605                                 match, mystat = _check_distfile(
 606                                         myfile_path, pruned_digests, eout)
 607                                 if match:
 608                                         if distdir_writable:
 609                                                 try:
 610                                                         apply_secpass_permissions(myfile_path,
 611                                                                 gid=portage_gid, mode=0o664, mask=0o2,
 612                                                                 stat_cached=mystat)
 613                                                 except PortageException as e:
 614                                                         if not os.access(myfile_path, os.R_OK):
 615                                                                 writemsg(_("!!! Failed to adjust permissions:"
 616                                                                         " %s\n") % str(e), noiselevel=-1)
 617                                                         del e
 618                                         continue
 619
 620                                 if distdir_writable and mystat is None:
 621                                         # Remove broken symlinks if necessary.
 622                                         try:
 623                                                 os.unlink(myfile_path)
 624                                         except OSError:
 625                                                 pass
 626
 627                                 if mystat is not None:
 628                                         if stat.S_ISDIR(mystat.st_mode):
 629                                                 writemsg_level(
 630                                                         _("!!! Unable to fetch file since "
 631                                                         "a directory is in the way: \n"
 632                                                         "!!!   %s\n") % myfile_path,
 633                                                         level=logging.ERROR, noiselevel=-1)
 634                                                 return 0
 635
 636                                         if mystat.st_size == 0:
 637                                                 if distdir_writable:
 638                                                         try:
 639                                                                 os.unlink(myfile_path)
 640                                                         except OSError:
 641                                                                 pass
 642                                         elif distdir_writable:
 643                                                 if mystat.st_size < fetch_resume_size and \
 644                                                         mystat.st_size < size:
 645                                                         # If the file already exists and the size does not
 646                                                         # match the existing digests, it may be that the
 647                                                         # user is attempting to update the digest. In this
 648                                                         # case, the digestgen() function will advise the
 649                                                         # user to use `ebuild --force foo.ebuild manifest`
 650                                                         # in order to force the old digests to be replaced.
 651                                                         # Since the user may want to keep this file, rename
 652                                                         # it instead of deleting it.
 653                                                         writemsg(_(">>> Renaming distfile with size "
 654                                                                 "%d (smaller than " "PORTAGE_FETCH_RESU"
 655                                                                 "ME_MIN_SIZE)\n") % mystat.st_size)
 656                                                         temp_filename = \
 657                                                                 _checksum_failure_temp_file(
 658                                                                 mysettings["DISTDIR"], myfile)
 659                                                         writemsg_stdout(_("Refetching... "
 660                                                                 "File renamed to '%s'\n\n") % \
 661                                                                 temp_filename, noiselevel=-1)
 662                                                 elif mystat.st_size >= size:
 663                                                         temp_filename = \
 664                                                                 _checksum_failure_temp_file(
 665                                                                 mysettings["DISTDIR"], myfile)
 666                                                         writemsg_stdout(_("Refetching... "
 667                                                                 "File renamed to '%s'\n\n") % \
 668                                                                 temp_filename, noiselevel=-1)
 669
 670                                 if distdir_writable and ro_distdirs:
 671                                         readonly_file = None
 672                                         for x in ro_distdirs:
 673                                                 filename = os.path.join(x, myfile)
 674                                                 match, mystat = _check_distfile(
 675                                                         filename, pruned_digests, eout)
 676                                                 if match:
 677                                                         readonly_file = filename
 678                                                         break
 679                                         if readonly_file is not None:
 680                                                 try:
 681                                                         os.unlink(myfile_path)
 682                                                 except OSError as e:
 683                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 684                                                                 raise
 685                                                         del e
 686                                                 os.symlink(readonly_file, myfile_path)
 687                                                 continue
 688
 689                                 if fsmirrors and not os.path.exists(myfile_path) and has_space:
 690                                         for mydir in fsmirrors:
 691                                                 mirror_file = os.path.join(mydir, myfile)
 692                                                 try:
 693                                                         shutil.copyfile(mirror_file, myfile_path)
 694                                                         writemsg(_("Local mirror has file: %s\n") % myfile)
 695                                                         break
 696                                                 except (IOError, OSError) as e:
 697                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 698                                                                 raise
 699                                                         del e
 700
 701                                 try:
 702                                         mystat = os.stat(myfile_path)
 703                                 except OSError as e:
 704                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 705                                                 raise
 706                                         del e
 707                                 else:
 708                                         try:
 709                                                 apply_secpass_permissions(
 710                                                         myfile_path, gid=portage_gid, mode=0o664, mask=0o2,
 711                                                         stat_cached=mystat)
 712                                         except PortageException as e:
 713                                                 if not os.access(myfile_path, os.R_OK):
 714                                                         writemsg(_("!!! Failed to adjust permissions:"
 715                                                                 " %s\n") % str(e), noiselevel=-1)
 716
 717                                         # If the file is empty then it's obviously invalid. Remove
 718                                         # the empty file and try to download if possible.
 719                                         if mystat.st_size == 0:
 720                                                 if distdir_writable:
 721                                                         try:
 722                                                                 os.unlink(myfile_path)
 723                                                         except EnvironmentError:
 724                                                                 pass
 725                                         elif myfile not in mydigests:
 726                                                 # We don't have a digest, but the file exists.  We must
 727                                                 # assume that it is fully downloaded.
 728                                                 continue
 729                                         else:
 730                                                 if mystat.st_size < mydigests[myfile]["size"] and \
 731                                                         not restrict_fetch:
 732                                                         fetched = 1 # Try to resume this download.
 733                                                 elif parallel_fetchonly and \
 734                                                         mystat.st_size == mydigests[myfile]["size"]:
 735                                                         eout = EOutput()
 736                                                         eout.quiet = \
 737                                                                 mysettings.get("PORTAGE_QUIET") == "1"
 738                                                         eout.ebegin(
 739                                                                 "%s size ;-)" % (myfile, ))
 740                                                         eout.eend(0)
 741                                                         continue
 742                                                 else:
 743                                                         verified_ok, reason = verify_all(
 744                                                                 myfile_path, mydigests[myfile])
 745                                                         if not verified_ok:
 746                                                                 writemsg(_("!!! Previously fetched"
 747                                                                         " file: '%s'\n") % myfile, noiselevel=-1)
 748                                                                 writemsg(_("!!! Reason: %s\n") % reason[0],
 749                                                                         noiselevel=-1)
 750                                                                 writemsg(_("!!! Got:      %s\n"
 751                                                                         "!!! Expected: %s\n") % \
 752                                                                         (reason[1], reason[2]), noiselevel=-1)
 753                                                                 if reason[0] == _("Insufficient data for checksum verification"):
 754                                                                         return 0
 755                                                                 if distdir_writable:
 756                                                                         temp_filename = \
 757                                                                                 _checksum_failure_temp_file(
 758                                                                                 mysettings["DISTDIR"], myfile)
 759                                                                         writemsg_stdout(_("Refetching... "
 760                                                                                 "File renamed to '%s'\n\n") % \
 761                                                                                 temp_filename, noiselevel=-1)
 762                                                         else:
 763                                                                 eout = EOutput()
 764                                                                 eout.quiet = \
 765                                                                         mysettings.get("PORTAGE_QUIET", None) == "1"
 766                                                                 digests = mydigests.get(myfile)
 767                                                                 if digests:
 768                                                                         digests = list(digests)
 769                                                                         digests.sort()
 770                                                                         eout.ebegin(
 771                                                                                 "%s %s ;-)" % (myfile, " ".join(digests)))
 772                                                                         eout.eend(0)
 773                                                                 continue # fetch any remaining files
 774
 775                         # Create a reversed list since that is optimal for list.pop().
 776                         uri_list = filedict[myfile][:]
 777                         uri_list.reverse()
 778                         checksum_failure_count = 0
 779                         tried_locations = set()
 780                         while uri_list:
 781                                 loc = uri_list.pop()
 782                                 # Eliminate duplicates here in case we've switched to
 783                                 # "primaryuri" mode on the fly due to a checksum failure.
 784                                 if loc in tried_locations:
 785                                         continue
 786                                 tried_locations.add(loc)
 787                                 if listonly:
 788                                         writemsg_stdout(loc+" ", noiselevel=-1)
 789                                         continue
 790                                 # allow different fetchcommands per protocol
 791                                 protocol = loc[0:loc.find("://")]
 792
 793                                 global_config_path = GLOBAL_CONFIG_PATH
 794                                 if mysettings['EPREFIX']:
 795                                         global_config_path = os.path.join(mysettings['EPREFIX'],
 796                                                         GLOBAL_CONFIG_PATH.lstrip(os.sep))
 797
 798                                 missing_file_param = False
 799                                 fetchcommand_var = "FETCHCOMMAND_" + protocol.upper()
 800                                 fetchcommand = mysettings.get(fetchcommand_var)
 801                                 if fetchcommand is None:
 802                                         fetchcommand_var = "FETCHCOMMAND"
 803                                         fetchcommand = mysettings.get(fetchcommand_var)
 804                                         if fetchcommand is None:
 805                                                 writemsg_level(
 806                                                         _("!!! %s is unset. It should "
 807                                                         "have been defined in\n!!! %s/make.globals.\n") \
 808                                                         % (fetchcommand_var, global_config_path),
 809                                                         level=logging.ERROR, noiselevel=-1)
 810                                                 return 0
 811                                 if "${FILE}" not in fetchcommand:
 812                                         writemsg_level(
 813                                                 _("!!! %s does not contain the required ${FILE}"
 814                                                 " parameter.\n") % fetchcommand_var,
 815                                                 level=logging.ERROR, noiselevel=-1)
 816                                         missing_file_param = True
 817
 818                                 resumecommand_var = "RESUMECOMMAND_" + protocol.upper()
 819                                 resumecommand = mysettings.get(resumecommand_var)
 820                                 if resumecommand is None:
 821                                         resumecommand_var = "RESUMECOMMAND"
 822                                         resumecommand = mysettings.get(resumecommand_var)
 823                                         if resumecommand is None:
 824                                                 writemsg_level(
 825                                                         _("!!! %s is unset. It should "
 826                                                         "have been defined in\n!!! %s/make.globals.\n") \
 827                                                         % (resumecommand_var, global_config_path),
 828                                                         level=logging.ERROR, noiselevel=-1)
 829                                                 return 0
 830                                 if "${FILE}" not in resumecommand:
 831                                         writemsg_level(
 832                                                 _("!!! %s does not contain the required ${FILE}"
 833                                                 " parameter.\n") % resumecommand_var,
 834                                                 level=logging.ERROR, noiselevel=-1)
 835                                         missing_file_param = True
 836
 837                                 if missing_file_param:
 838                                         writemsg_level(
 839                                                 _("!!! Refer to the make.conf(5) man page for "
 840                                                 "information about how to\n!!! correctly specify "
 841                                                 "FETCHCOMMAND and RESUMECOMMAND.\n"),
 842                                                 level=logging.ERROR, noiselevel=-1)
 843                                         if myfile != os.path.basename(loc):
 844                                                 return 0
 845
 846                                 if not can_fetch:
 847                                         if fetched != 2:
 848                                                 try:
 849                                                         mysize = os.stat(myfile_path).st_size
 850                                                 except OSError as e:
 851                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 852                                                                 raise
 853                                                         del e
 854                                                         mysize = 0
 855
 856                                                 if mysize == 0:
 857                                                         writemsg(_("!!! File %s isn't fetched but unable to get it.\n") % myfile,
 858                                                                 noiselevel=-1)
 859                                                 elif size is None or size > mysize:
 860                                                         writemsg(_("!!! File %s isn't fully fetched, but unable to complete it\n") % myfile,
 861                                                                 noiselevel=-1)
 862                                                 else:
 863                                                         writemsg(_("!!! File %s is incorrect size, "
 864                                                                 "but unable to retry.\n") % myfile, noiselevel=-1)
 865                                                 return 0
 866                                         else:
 867                                                 continue
 868
 869                                 if fetched != 2 and has_space:
 870                                         #we either need to resume or start the download
 871                                         if fetched == 1:
 872                                                 try:
 873                                                         mystat = os.stat(myfile_path)
 874                                                 except OSError as e:
 875                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 876                                                                 raise
 877                                                         del e
 878                                                         fetched = 0
 879                                                 else:
 880                                                         if mystat.st_size < fetch_resume_size:
 881                                                                 writemsg(_(">>> Deleting distfile with size "
 882                                                                         "%d (smaller than " "PORTAGE_FETCH_RESU"
 883                                                                         "ME_MIN_SIZE)\n") % mystat.st_size)
 884                                                                 try:
 885                                                                         os.unlink(myfile_path)
 886                                                                 except OSError as e:
 887                                                                         if e.errno not in \
 888                                                                                 (errno.ENOENT, errno.ESTALE):
 889                                                                                 raise
 890                                                                         del e
 891                                                                 fetched = 0
 892                                         if fetched == 1:
 893                                                 #resume mode:
 894                                                 writemsg(_(">>> Resuming download...\n"))
 895                                                 locfetch=resumecommand
 896                                                 command_var = resumecommand_var
 897                                         else:
 898                                                 #normal mode:
 899                                                 locfetch=fetchcommand
 900                                                 command_var = fetchcommand_var
 901                                         writemsg_stdout(_(">>> Downloading '%s'\n") % \
 902                                                 re.sub(r'//(.+):.+@(.+)/',r'//\1:*password*@\2/', loc))
 903                                         variables = {
 904                                                 "DISTDIR": mysettings["DISTDIR"],
 905                                                 "URI":     loc,
 906                                                 "FILE":    myfile
 907                                         }
 908
 909                                         myfetch = shlex_split(locfetch)
 910                                         myfetch = [varexpand(x, mydict=variables) for x in myfetch]
 911                                         myret = -1
 912                                         try:
 913
 914                                                 myret = _spawn_fetch(mysettings, myfetch)
 915
 916                                         finally:
 917                                                 try:
 918                                                         apply_secpass_permissions(myfile_path,
 919                                                                 gid=portage_gid, mode=0o664, mask=0o2)
 920                                                 except FileNotFound:
 921                                                         pass
 922                                                 except PortageException as e:
 923                                                         if not os.access(myfile_path, os.R_OK):
 924                                                                 writemsg(_("!!! Failed to adjust permissions:"
 925                                                                         " %s\n") % str(e), noiselevel=-1)
 926                                                         del e
 927
 928                                         # If the file is empty then it's obviously invalid.  Don't
 929                                         # trust the return value from the fetcher.  Remove the
 930                                         # empty file and try to download again.
 931                                         try:
 932                                                 if os.stat(myfile_path).st_size == 0:
 933                                                         os.unlink(myfile_path)
 934                                                         fetched = 0
 935                                                         continue
 936                                         except EnvironmentError:
 937                                                 pass
 938
 939                                         if mydigests is not None and myfile in mydigests:
 940                                                 try:
 941                                                         mystat = os.stat(myfile_path)
 942                                                 except OSError as e:
 943                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 944                                                                 raise
 945                                                         del e
 946                                                         fetched = 0
 947                                                 else:
 948
 949                                                         if stat.S_ISDIR(mystat.st_mode):
 950                                                                 # This can happen if FETCHCOMMAND erroneously
 951                                                                 # contains wget's -P option where it should
 952                                                                 # instead have -O.
 953                                                                 writemsg_level(
 954                                                                         _("!!! The command specified in the "
 955                                                                         "%s variable appears to have\n!!! "
 956                                                                         "created a directory instead of a "
 957                                                                         "normal file.\n") % command_var,
 958                                                                         level=logging.ERROR, noiselevel=-1)
 959                                                                 writemsg_level(
 960                                                                         _("!!! Refer to the make.conf(5) "
 961                                                                         "man page for information about how "
 962                                                                         "to\n!!! correctly specify "
 963                                                                         "FETCHCOMMAND and RESUMECOMMAND.\n"),
 964                                                                         level=logging.ERROR, noiselevel=-1)
 965                                                                 return 0
 966
 967                                                         # no exception?  file exists. let digestcheck() report
 968                                                         # an appropriately for size or checksum errors
 969
 970                                                         # If the fetcher reported success and the file is
 971                                                         # too small, it's probably because the digest is
 972                                                         # bad (upstream changed the distfile).  In this
 973                                                         # case we don't want to attempt to resume. Show a
 974                                                         # digest verification failure to that the user gets
 975                                                         # a clue about what just happened.
 976                                                         if myret != os.EX_OK and \
 977                                                                 mystat.st_size < mydigests[myfile]["size"]:
 978                                                                 # Fetch failed... Try the next one... Kill 404 files though.
 979                                                                 if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
 980                                                                         html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M)
 981                                                                         if html404.search(codecs.open(
 982                                                                                 _unicode_encode(myfile_path,
 983                                                                                 encoding=_encodings['fs'], errors='strict'),
 984                                                                                 mode='r', encoding=_encodings['content'], errors='replace'
 985                                                                                 ).read()):
 986                                                                                 try:
 987                                                                                         os.unlink(mysettings["DISTDIR"]+"/"+myfile)
 988                                                                                         writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n"))
 989                                                                                         fetched = 0
 990                                                                                         continue
 991                                                                                 except (IOError, OSError):
 992                                                                                         pass
 993                                                                 fetched = 1
 994                                                                 continue
 995                                                         if True:
 996                                                                 # File is the correct size--check the checksums for the fetched
 997                                                                 # file NOW, for those users who don't have a stable/continuous
 998                                                                 # net connection. This way we have a chance to try to download
 999                                                                 # from another mirror...
1000                                                                 verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile])
1001                                                                 if not verified_ok:
1002                                                                         print(reason)
1003                                                                         writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
1004                                                                                 noiselevel=-1)
1005                                                                         writemsg(_("!!! Reason: %s\n") % reason[0],
1006                                                                                 noiselevel=-1)
1007                                                                         writemsg(_("!!! Got:      %s\n!!! Expected: %s\n") % \
1008                                                                                 (reason[1], reason[2]), noiselevel=-1)
1009                                                                         if reason[0] == _("Insufficient data for checksum verification"):
1010                                                                                 return 0
1011                                                                         temp_filename = \
1012                                                                                 _checksum_failure_temp_file(
1013                                                                                 mysettings["DISTDIR"], myfile)
1014                                                                         writemsg_stdout(_("Refetching... "
1015                                                                                 "File renamed to '%s'\n\n") % \
1016                                                                                 temp_filename, noiselevel=-1)
1017                                                                         fetched=0
1018                                                                         checksum_failure_count += 1
1019                                                                         if checksum_failure_count == \
1020                                                                                 checksum_failure_primaryuri:
1021                                                                                 # Switch to "primaryuri" mode in order
1022                                                                                 # to increase the probablility of
1023                                                                                 # of success.
1024                                                                                 primaryuris = \
1025                                                                                         primaryuri_dict.get(myfile)
1026                                                                                 if primaryuris:
1027                                                                                         uri_list.extend(
1028                                                                                                 reversed(primaryuris))
1029                                                                         if checksum_failure_count >= \
1030                                                                                 checksum_failure_max_tries:
1031                                                                                 break
1032                                                                 else:
1033                                                                         eout = EOutput()
1034                                                                         eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
1035                                                                         digests = mydigests.get(myfile)
1036                                                                         if digests:
1037                                                                                 eout.ebegin("%s %s ;-)" % \
1038                                                                                         (myfile, " ".join(sorted(digests))))
1039                                                                                 eout.eend(0)
1040                                                                         fetched=2
1041                                                                         break
1042                                         else:
1043                                                 if not myret:
1044                                                         fetched=2
1045                                                         break
1046                                                 elif mydigests!=None:
1047                                                         writemsg(_("No digest file available and download failed.\n\n"),
1048                                                                 noiselevel=-1)
1049                 finally:
1050                         if use_locks and file_lock:
1051                                 unlockfile(file_lock)
1052                                 file_lock = None
1053
1054                 if listonly:
1055                         writemsg_stdout("\n", noiselevel=-1)
1056                 if fetched != 2:
1057                         if restrict_fetch and not restrict_fetch_msg:
1058                                 restrict_fetch_msg = True
1059                                 msg = _("\n!!! %s/%s"
1060                                         " has fetch restriction turned on.\n"
1061                                         "!!! This probably means that this "
1062                                         "ebuild's files must be downloaded\n"
1063                                         "!!! manually.  See the comments in"
1064                                         " the ebuild for more information.\n\n") % \
1065                                         (mysettings["CATEGORY"], mysettings["PF"])
1066                                 writemsg_level(msg,
1067                                         level=logging.ERROR, noiselevel=-1)
1068                         elif restrict_fetch:
1069                                 pass
1070                         elif listonly:
1071                                 pass
1072                         elif not filedict[myfile]:
1073                                 writemsg(_("Warning: No mirrors available for file"
1074                                         " '%s'\n") % (myfile), noiselevel=-1)
1075                         else:
1076                                 writemsg(_("!!! Couldn't download '%s'. Aborting.\n") % myfile,
1077                                         noiselevel=-1)
1078
1079                         if listonly:
1080                                 continue
1081                         elif fetchonly:
1082                                 failed_files.add(myfile)
1083                                 continue
1084                         return 0
1085         if failed_files:
1086                 return 0
1087         return 1