pym/portage/package/ebuild/fetch.py

   1 # Copyright 2010-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import print_function
   5
   6 __all__ = ['fetch']
   7
   8 import errno
   9 import io
  10 import logging
  11 import random
  12 import re
  13 import stat
  14 import sys
  15 import tempfile
  16
  17 try:
  18         from urllib.parse import urlparse
  19 except ImportError:
  20         from urlparse import urlparse
  21
  22 import portage
  23 portage.proxy.lazyimport.lazyimport(globals(),
  24         'portage.package.ebuild.config:check_config_instance,config',
  25         'portage.package.ebuild.doebuild:doebuild_environment,' + \
  26                 '_doebuild_spawn',
  27         'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
  28 )
  29
  30 from portage import OrderedDict, os, selinux, shutil, _encodings, \
  31         _shell_quote, _unicode_encode
  32 from portage.checksum import (hashfunc_map, perform_md5, verify_all,
  33         _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter)
  34 from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
  35         GLOBAL_CONFIG_PATH
  36 from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
  37 from portage.exception import FileNotFound, OperationNotPermitted, \
  38         PortageException, TryAgain
  39 from portage.localization import _
  40 from portage.locks import lockfile, unlockfile
  41 from portage.output import colorize, EOutput
  42 from portage.util import apply_recursive_permissions, \
  43         apply_secpass_permissions, ensure_dirs, grabdict, shlex_split, \
  44         varexpand, writemsg, writemsg_level, writemsg_stdout
  45 from portage.process import spawn
  46
  47 _userpriv_spawn_kwargs = (
  48         ("uid",    portage_uid),
  49         ("gid",    portage_gid),
  50         ("groups", userpriv_groups),
  51         ("umask",  0o02),
  52 )
  53
  54 def _hide_url_passwd(url):
  55         return re.sub(r'//(.+):.+@(.+)', r'//\1:*password*@\2', url)
  56
  57 def _spawn_fetch(settings, args, **kwargs):
  58         """
  59         Spawn a process with appropriate settings for fetching, including
  60         userfetch and selinux support.
  61         """
  62
  63         global _userpriv_spawn_kwargs
  64
  65         # Redirect all output to stdout since some fetchers like
  66         # wget pollute stderr (if portage detects a problem then it
  67         # can send it's own message to stderr).
  68         if "fd_pipes" not in kwargs:
  69
  70                 kwargs["fd_pipes"] = {
  71                         0 : portage._get_stdin().fileno(),
  72                         1 : sys.__stdout__.fileno(),
  73                         2 : sys.__stdout__.fileno(),
  74                 }
  75
  76         if "userfetch" in settings.features and \
  77                 os.getuid() == 0 and portage_gid and portage_uid and \
  78                 hasattr(os, "setgroups"):
  79                 kwargs.update(_userpriv_spawn_kwargs)
  80
  81         spawn_func = spawn
  82
  83         if settings.selinux_enabled():
  84                 spawn_func = selinux.spawn_wrapper(spawn_func,
  85                         settings["PORTAGE_FETCH_T"])
  86
  87                 # bash is an allowed entrypoint, while most binaries are not
  88                 if args[0] != BASH_BINARY:
  89                         args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
  90
  91         # Ensure that EBUILD_PHASE is set to fetch, so that config.environ()
  92         # does not filter the calling environment (which may contain needed
  93         # proxy variables, as in bug #315421).
  94         phase_backup = settings.get('EBUILD_PHASE')
  95         settings['EBUILD_PHASE'] = 'fetch'
  96         try:
  97                 rval = spawn_func(args, env=settings.environ(), **kwargs)
  98         finally:
  99                 if phase_backup is None:
 100                         settings.pop('EBUILD_PHASE', None)
 101                 else:
 102                         settings['EBUILD_PHASE'] = phase_backup
 103
 104         return rval
 105
 106 _userpriv_test_write_file_cache = {}
 107 _userpriv_test_write_cmd_script = ">> %(file_path)s 2>/dev/null ; rval=$? ; " + \
 108         "rm -f  %(file_path)s ; exit $rval"
 109
 110 def _userpriv_test_write_file(settings, file_path):
 111         """
 112         Drop privileges and try to open a file for writing. The file may or
 113         may not exist, and the parent directory is assumed to exist. The file
 114         is removed before returning.
 115
 116         @param settings: A config instance which is passed to _spawn_fetch()
 117         @param file_path: A file path to open and write.
 118         @return: True if write succeeds, False otherwise.
 119         """
 120
 121         global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script
 122         rval = _userpriv_test_write_file_cache.get(file_path)
 123         if rval is not None:
 124                 return rval
 125
 126         args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \
 127                 {"file_path" : _shell_quote(file_path)}]
 128
 129         returncode = _spawn_fetch(settings, args)
 130
 131         rval = returncode == os.EX_OK
 132         _userpriv_test_write_file_cache[file_path] = rval
 133         return rval
 134
 135 def _checksum_failure_temp_file(distdir, basename):
 136         """
 137         First try to find a duplicate temp file with the same checksum and return
 138         that filename if available. Otherwise, use mkstemp to create a new unique
 139         filename._checksum_failure_.$RANDOM, rename the given file, and return the
 140         new filename. In any case, filename will be renamed or removed before this
 141         function returns a temp filename.
 142         """
 143
 144         filename = os.path.join(distdir, basename)
 145         size = os.stat(filename).st_size
 146         checksum = None
 147         tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*')
 148         for temp_filename in os.listdir(distdir):
 149                 if not tempfile_re.match(temp_filename):
 150                         continue
 151                 temp_filename = os.path.join(distdir, temp_filename)
 152                 try:
 153                         if size != os.stat(temp_filename).st_size:
 154                                 continue
 155                 except OSError:
 156                         continue
 157                 try:
 158                         temp_checksum = perform_md5(temp_filename)
 159                 except FileNotFound:
 160                         # Apparently the temp file disappeared. Let it go.
 161                         continue
 162                 if checksum is None:
 163                         checksum = perform_md5(filename)
 164                 if checksum == temp_checksum:
 165                         os.unlink(filename)
 166                         return temp_filename
 167
 168         fd, temp_filename = \
 169                 tempfile.mkstemp("", basename + "._checksum_failure_.", distdir)
 170         os.close(fd)
 171         os.rename(filename, temp_filename)
 172         return temp_filename
 173
 174 def _check_digests(filename, digests, show_errors=1):
 175         """
 176         Check digests and display a message if an error occurs.
 177         @return True if all digests match, False otherwise.
 178         """
 179         verified_ok, reason = verify_all(filename, digests)
 180         if not verified_ok:
 181                 if show_errors:
 182                         writemsg(_("!!! Previously fetched"
 183                                 " file: '%s'\n") % filename, noiselevel=-1)
 184                         writemsg(_("!!! Reason: %s\n") % reason[0],
 185                                 noiselevel=-1)
 186                         writemsg(_("!!! Got:      %s\n"
 187                                 "!!! Expected: %s\n") % \
 188                                 (reason[1], reason[2]), noiselevel=-1)
 189                 return False
 190         return True
 191
 192 def _check_distfile(filename, digests, eout, show_errors=1, hash_filter=None):
 193         """
 194         @return a tuple of (match, stat_obj) where match is True if filename
 195         matches all given digests (if any) and stat_obj is a stat result, or
 196         None if the file does not exist.
 197         """
 198         if digests is None:
 199                 digests = {}
 200         size = digests.get("size")
 201         if size is not None and len(digests) == 1:
 202                 digests = None
 203
 204         try:
 205                 st = os.stat(filename)
 206         except OSError:
 207                 return (False, None)
 208         if size is not None and size != st.st_size:
 209                 return (False, st)
 210         if not digests:
 211                 if size is not None:
 212                         eout.ebegin(_("%s size ;-)") % os.path.basename(filename))
 213                         eout.eend(0)
 214                 elif st.st_size == 0:
 215                         # Zero-byte distfiles are always invalid.
 216                         return (False, st)
 217         else:
 218                 digests = _filter_unaccelarated_hashes(digests)
 219                 if hash_filter is not None:
 220                         digests = _apply_hash_filter(digests, hash_filter)
 221                 if _check_digests(filename, digests, show_errors=show_errors):
 222                         eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
 223                                 " ".join(sorted(digests))))
 224                         eout.eend(0)
 225                 else:
 226                         return (False, st)
 227         return (True, st)
 228
 229 _fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
 230
 231 _size_suffix_map = {
 232         ''  : 0,
 233         'K' : 10,
 234         'M' : 20,
 235         'G' : 30,
 236         'T' : 40,
 237         'P' : 50,
 238         'E' : 60,
 239         'Z' : 70,
 240         'Y' : 80,
 241 }
 242
 243
 244 def _get_checksum_failure_max_tries(settings, default=5):
 245         """
 246         Get the maximum number of failed download attempts.
 247
 248         Generally, downloading the same file repeatedly from
 249         every single available mirror is a waste of bandwidth
 250         and time, so there needs to be a cap.
 251         """
 252         key = 'PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS'
 253         v = default
 254         try:
 255                 v = int(settings.get(key, default))
 256         except (ValueError, OverflowError):
 257                 writemsg(_("!!! Variable %s contains "
 258                         "non-integer value: '%s'\n")
 259                         % (key, settings[key]),
 260                         noiselevel=-1)
 261                 writemsg(_("!!! Using %s default value: %s\n")
 262                         % (key, default),
 263                         noiselevel=-1)
 264                 v = default
 265         if v < 1:
 266                 writemsg(_("!!! Variable %s contains "
 267                         "value less than 1: '%s'\n")
 268                         % (key, v),
 269                         noiselevel=-1)
 270                 writemsg(_("!!! Using %s default value: %s\n")
 271                         % (key, default),
 272                         noiselevel=-1)
 273                 v = default
 274         return v
 275
 276
 277 def _get_fetch_resume_size(settings, default='350K'):
 278         key = 'PORTAGE_FETCH_RESUME_MIN_SIZE'
 279         v = settings.get(key)
 280         if v is not None:
 281                 v = "".join(v.split())
 282                 if not v:
 283                         # If it's empty, silently use the default.
 284                         v = default
 285                 match = _fetch_resume_size_re.match(v)
 286                 if (match is None or
 287                                 match.group(2).upper() not in _size_suffix_map):
 288                         writemsg(_("!!! Variable %s contains an "
 289                                 "unrecognized format: '%s'\n")
 290                                 % (key, settings[key]),
 291                                 noiselevel=-1)
 292                         writemsg(_("!!! Using %s default value: %s\n")
 293                                 % (key, default),
 294                                 noiselevel=-1)
 295                         v = None
 296         if v is None:
 297                 v = default
 298                 match = _fetch_resume_size_re.match(v)
 299         v = int(match.group(1)) * \
 300                 2 ** _size_suffix_map[match.group(2).upper()]
 301         return v
 302
 303
 304 def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 305         locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
 306         allow_missing_digests=True):
 307         "fetch files.  Will use digest file if available."
 308
 309         if not myuris:
 310                 return 1
 311
 312         features = mysettings.features
 313         restrict = mysettings.get("PORTAGE_RESTRICT","").split()
 314
 315         userfetch = secpass >= 2 and "userfetch" in features
 316         userpriv = secpass >= 2 and "userpriv" in features
 317
 318         # 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring.
 319         restrict_mirror = "mirror" in restrict or "nomirror" in restrict
 320         if restrict_mirror:
 321                 if ("mirror" in features) and ("lmirror" not in features):
 322                         # lmirror should allow you to bypass mirror restrictions.
 323                         # XXX: This is not a good thing, and is temporary at best.
 324                         print(_(">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch."))
 325                         return 1
 326
 327         checksum_failure_max_tries = _get_checksum_failure_max_tries(
 328                 settings=mysettings)
 329         fetch_resume_size = _get_fetch_resume_size(settings=mysettings)
 330
 331         # Behave like the package has RESTRICT="primaryuri" after a
 332         # couple of checksum failures, to increase the probablility
 333         # of success before checksum_failure_max_tries is reached.
 334         checksum_failure_primaryuri = 2
 335         thirdpartymirrors = mysettings.thirdpartymirrors()
 336
 337         # In the background parallel-fetch process, it's safe to skip checksum
 338         # verification of pre-existing files in $DISTDIR that have the correct
 339         # file size. The parent process will verify their checksums prior to
 340         # the unpack phase.
 341
 342         parallel_fetchonly = "PORTAGE_PARALLEL_FETCHONLY" in mysettings
 343         if parallel_fetchonly:
 344                 fetchonly = 1
 345
 346         check_config_instance(mysettings)
 347
 348         custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
 349                 CUSTOM_MIRRORS_FILE), recursive=1)
 350
 351         mymirrors=[]
 352
 353         if listonly or ("distlocks" not in features):
 354                 use_locks = 0
 355
 356         fetch_to_ro = 0
 357         if "skiprocheck" in features:
 358                 fetch_to_ro = 1
 359
 360         if not os.access(mysettings["DISTDIR"],os.W_OK) and fetch_to_ro:
 361                 if use_locks:
 362                         writemsg(colorize("BAD",
 363                                 _("!!! For fetching to a read-only filesystem, "
 364                                 "locking should be turned off.\n")), noiselevel=-1)
 365                         writemsg(_("!!! This can be done by adding -distlocks to "
 366                                 "FEATURES in /etc/portage/make.conf\n"), noiselevel=-1)
 367 #                       use_locks = 0
 368
 369         # local mirrors are always added
 370         if "local" in custommirrors:
 371                 mymirrors += custommirrors["local"]
 372
 373         if restrict_mirror:
 374                 # We don't add any mirrors.
 375                 pass
 376         else:
 377                 if try_mirrors:
 378                         mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
 379
 380         hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
 381         if hash_filter.transparent:
 382                 hash_filter = None
 383         skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
 384         if skip_manifest:
 385                 allow_missing_digests = True
 386         pkgdir = mysettings.get("O")
 387         if digests is None and not (pkgdir is None or skip_manifest):
 388                 mydigests = mysettings.repositories.get_repo_for_location(
 389                         os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
 390                         pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
 391         elif digests is None or skip_manifest:
 392                 # no digests because fetch was not called for a specific package
 393                 mydigests = {}
 394         else:
 395                 mydigests = digests
 396
 397         ro_distdirs = [x for x in \
 398                 shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
 399                 if os.path.isdir(x)]
 400
 401         fsmirrors = []
 402         for x in range(len(mymirrors)-1,-1,-1):
 403                 if mymirrors[x] and mymirrors[x][0]=='/':
 404                         fsmirrors += [mymirrors[x]]
 405                         del mymirrors[x]
 406
 407         restrict_fetch = "fetch" in restrict
 408         force_mirror = "force-mirror" in features and not restrict_mirror
 409         custom_local_mirrors = custommirrors.get("local", [])
 410         if restrict_fetch:
 411                 # With fetch restriction, a normal uri may only be fetched from
 412                 # custom local mirrors (if available).  A mirror:// uri may also
 413                 # be fetched from specific mirrors (effectively overriding fetch
 414                 # restriction, but only for specific mirrors).
 415                 locations = custom_local_mirrors
 416         else:
 417                 locations = mymirrors
 418
 419         file_uri_tuples = []
 420         # Check for 'items' attribute since OrderedDict is not a dict.
 421         if hasattr(myuris, 'items'):
 422                 for myfile, uri_set in myuris.items():
 423                         for myuri in uri_set:
 424                                 file_uri_tuples.append((myfile, myuri))
 425                         if not uri_set:
 426                                 file_uri_tuples.append((myfile, None))
 427         else:
 428                 for myuri in myuris:
 429                         if urlparse(myuri).scheme:
 430                                 file_uri_tuples.append((os.path.basename(myuri), myuri))
 431                         else:
 432                                 file_uri_tuples.append((os.path.basename(myuri), None))
 433
 434         filedict = OrderedDict()
 435         primaryuri_dict = {}
 436         thirdpartymirror_uris = {}
 437         for myfile, myuri in file_uri_tuples:
 438                 if myfile not in filedict:
 439                         filedict[myfile]=[]
 440                         for y in range(0,len(locations)):
 441                                 filedict[myfile].append(locations[y]+"/distfiles/"+myfile)
 442                 if myuri is None:
 443                         continue
 444                 if myuri[:9]=="mirror://":
 445                         eidx = myuri.find("/", 9)
 446                         if eidx != -1:
 447                                 mirrorname = myuri[9:eidx]
 448                                 path = myuri[eidx+1:]
 449
 450                                 # Try user-defined mirrors first
 451                                 if mirrorname in custommirrors:
 452                                         for cmirr in custommirrors[mirrorname]:
 453                                                 filedict[myfile].append(
 454                                                         cmirr.rstrip("/") + "/" + path)
 455
 456                                 # now try the official mirrors
 457                                 if mirrorname in thirdpartymirrors:
 458                                         uris = [locmirr.rstrip("/") + "/" + path \
 459                                                 for locmirr in thirdpartymirrors[mirrorname]]
 460                                         random.shuffle(uris)
 461                                         filedict[myfile].extend(uris)
 462                                         thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
 463
 464                                 if not filedict[myfile]:
 465                                         writemsg(_("No known mirror by the name: %s\n") % (mirrorname))
 466                         else:
 467                                 writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1)
 468                                 writemsg("  %s\n" % (myuri), noiselevel=-1)
 469                 else:
 470                         if restrict_fetch or force_mirror:
 471                                 # Only fetch from specific mirrors is allowed.
 472                                 continue
 473                         primaryuris = primaryuri_dict.get(myfile)
 474                         if primaryuris is None:
 475                                 primaryuris = []
 476                                 primaryuri_dict[myfile] = primaryuris
 477                         primaryuris.append(myuri)
 478
 479         # Order primaryuri_dict values to match that in SRC_URI.
 480         for uris in primaryuri_dict.values():
 481                 uris.reverse()
 482
 483         # Prefer thirdpartymirrors over normal mirrors in cases when
 484         # the file does not yet exist on the normal mirrors.
 485         for myfile, uris in thirdpartymirror_uris.items():
 486                 primaryuri_dict.setdefault(myfile, []).extend(uris)
 487
 488         # Now merge primaryuri values into filedict (includes mirrors
 489         # explicitly referenced in SRC_URI).
 490         if "primaryuri" in restrict:
 491                 for myfile, uris in filedict.items():
 492                         filedict[myfile] = primaryuri_dict.get(myfile, []) + uris
 493         else:
 494                 for myfile in filedict:
 495                         filedict[myfile] += primaryuri_dict.get(myfile, [])
 496
 497         can_fetch=True
 498
 499         if listonly:
 500                 can_fetch = False
 501
 502         if can_fetch and not fetch_to_ro:
 503                 global _userpriv_test_write_file_cache
 504                 dirmode  = 0o070
 505                 filemode =   0o60
 506                 modemask =    0o2
 507                 dir_gid = portage_gid
 508                 if "FAKED_MODE" in mysettings:
 509                         # When inside fakeroot, directories with portage's gid appear
 510                         # to have root's gid. Therefore, use root's gid instead of
 511                         # portage's gid to avoid spurrious permissions adjustments
 512                         # when inside fakeroot.
 513                         dir_gid = 0
 514                 distdir_dirs = [""]
 515                 try:
 516
 517                         for x in distdir_dirs:
 518                                 mydir = os.path.join(mysettings["DISTDIR"], x)
 519                                 write_test_file = os.path.join(
 520                                         mydir, ".__portage_test_write__")
 521
 522                                 try:
 523                                         st = os.stat(mydir)
 524                                 except OSError:
 525                                         st = None
 526
 527                                 if st is not None and stat.S_ISDIR(st.st_mode):
 528                                         if not (userfetch or userpriv):
 529                                                 continue
 530                                         if _userpriv_test_write_file(mysettings, write_test_file):
 531                                                 continue
 532
 533                                 _userpriv_test_write_file_cache.pop(write_test_file, None)
 534                                 if ensure_dirs(mydir, gid=dir_gid, mode=dirmode, mask=modemask):
 535                                         if st is None:
 536                                                 # The directory has just been created
 537                                                 # and therefore it must be empty.
 538                                                 continue
 539                                         writemsg(_("Adjusting permissions recursively: '%s'\n") % mydir,
 540                                                 noiselevel=-1)
 541                                         def onerror(e):
 542                                                 raise # bail out on the first error that occurs during recursion
 543                                         if not apply_recursive_permissions(mydir,
 544                                                 gid=dir_gid, dirmode=dirmode, dirmask=modemask,
 545                                                 filemode=filemode, filemask=modemask, onerror=onerror):
 546                                                 raise OperationNotPermitted(
 547                                                         _("Failed to apply recursive permissions for the portage group."))
 548                 except PortageException as e:
 549                         if not os.path.isdir(mysettings["DISTDIR"]):
 550                                 writemsg("!!! %s\n" % str(e), noiselevel=-1)
 551                                 writemsg(_("!!! Directory Not Found: DISTDIR='%s'\n") % mysettings["DISTDIR"], noiselevel=-1)
 552                                 writemsg(_("!!! Fetching will fail!\n"), noiselevel=-1)
 553
 554         if can_fetch and \
 555                 not fetch_to_ro and \
 556                 not os.access(mysettings["DISTDIR"], os.W_OK):
 557                 writemsg(_("!!! No write access to '%s'\n") % mysettings["DISTDIR"],
 558                         noiselevel=-1)
 559                 can_fetch = False
 560
 561         distdir_writable = can_fetch and not fetch_to_ro
 562         failed_files = set()
 563         restrict_fetch_msg = False
 564
 565         for myfile in filedict:
 566                 """
 567                 fetched  status
 568                 0        nonexistent
 569                 1        partially downloaded
 570                 2        completely downloaded
 571                 """
 572                 fetched = 0
 573
 574                 orig_digests = mydigests.get(myfile, {})
 575
 576                 if not (allow_missing_digests or listonly):
 577                         verifiable_hash_types = set(orig_digests).intersection(hashfunc_map)
 578                         verifiable_hash_types.discard("size")
 579                         if not verifiable_hash_types:
 580                                 expected = set(hashfunc_map)
 581                                 expected.discard("size")
 582                                 expected = " ".join(sorted(expected))
 583                                 got = set(orig_digests)
 584                                 got.discard("size")
 585                                 got = " ".join(sorted(got))
 586                                 reason = (_("Insufficient data for checksum verification"),
 587                                         got, expected)
 588                                 writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
 589                                         noiselevel=-1)
 590                                 writemsg(_("!!! Reason: %s\n") % reason[0],
 591                                         noiselevel=-1)
 592                                 writemsg(_("!!! Got:      %s\n!!! Expected: %s\n") % \
 593                                         (reason[1], reason[2]), noiselevel=-1)
 594
 595                                 if fetchonly:
 596                                         failed_files.add(myfile)
 597                                         continue
 598                                 else:
 599                                         return 0
 600
 601                 size = orig_digests.get("size")
 602                 if size == 0:
 603                         # Zero-byte distfiles are always invalid, so discard their digests.
 604                         del mydigests[myfile]
 605                         orig_digests.clear()
 606                         size = None
 607                 pruned_digests = orig_digests
 608                 if parallel_fetchonly:
 609                         pruned_digests = {}
 610                         if size is not None:
 611                                 pruned_digests["size"] = size
 612
 613                 myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
 614                 has_space = True
 615                 has_space_superuser = True
 616                 file_lock = None
 617                 if listonly:
 618                         writemsg_stdout("\n", noiselevel=-1)
 619                 else:
 620                         # check if there is enough space in DISTDIR to completely store myfile
 621                         # overestimate the filesize so we aren't bitten by FS overhead
 622                         vfs_stat = None
 623                         if size is not None and hasattr(os, "statvfs"):
 624                                 try:
 625                                         vfs_stat = os.statvfs(mysettings["DISTDIR"])
 626                                 except OSError as e:
 627                                         writemsg_level("!!! statvfs('%s'): %s\n" %
 628                                                 (mysettings["DISTDIR"], e),
 629                                                 noiselevel=-1, level=logging.ERROR)
 630                                         del e
 631
 632                         if vfs_stat is not None:
 633                                 try:
 634                                         mysize = os.stat(myfile_path).st_size
 635                                 except OSError as e:
 636                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 637                                                 raise
 638                                         del e
 639                                         mysize = 0
 640                                 if (size - mysize + vfs_stat.f_bsize) >= \
 641                                         (vfs_stat.f_bsize * vfs_stat.f_bavail):
 642
 643                                         if (size - mysize + vfs_stat.f_bsize) >= \
 644                                                 (vfs_stat.f_bsize * vfs_stat.f_bfree):
 645                                                 has_space_superuser = False
 646
 647                                         if not has_space_superuser:
 648                                                 has_space = False
 649                                         elif secpass < 2:
 650                                                 has_space = False
 651                                         elif userfetch:
 652                                                 has_space = False
 653
 654                         if distdir_writable and use_locks:
 655
 656                                 lock_kwargs = {}
 657                                 if fetchonly:
 658                                         lock_kwargs["flags"] = os.O_NONBLOCK
 659
 660                                 try:
 661                                         file_lock = lockfile(myfile_path,
 662                                                 wantnewlockfile=1, **lock_kwargs)
 663                                 except TryAgain:
 664                                         writemsg(_(">>> File '%s' is already locked by "
 665                                                 "another fetcher. Continuing...\n") % myfile,
 666                                                 noiselevel=-1)
 667                                         continue
 668                 try:
 669                         if not listonly:
 670
 671                                 eout = EOutput()
 672                                 eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
 673                                 match, mystat = _check_distfile(
 674                                         myfile_path, pruned_digests, eout, hash_filter=hash_filter)
 675                                 if match:
 676                                         # Skip permission adjustment for symlinks, since we don't
 677                                         # want to modify anything outside of the primary DISTDIR,
 678                                         # and symlinks typically point to PORTAGE_RO_DISTDIRS.
 679                                         if distdir_writable and not os.path.islink(myfile_path):
 680                                                 try:
 681                                                         apply_secpass_permissions(myfile_path,
 682                                                                 gid=portage_gid, mode=0o664, mask=0o2,
 683                                                                 stat_cached=mystat)
 684                                                 except PortageException as e:
 685                                                         if not os.access(myfile_path, os.R_OK):
 686                                                                 writemsg(_("!!! Failed to adjust permissions:"
 687                                                                         " %s\n") % str(e), noiselevel=-1)
 688                                                         del e
 689                                         continue
 690
 691                                 if distdir_writable and mystat is None:
 692                                         # Remove broken symlinks if necessary.
 693                                         try:
 694                                                 os.unlink(myfile_path)
 695                                         except OSError:
 696                                                 pass
 697
 698                                 if mystat is not None:
 699                                         if stat.S_ISDIR(mystat.st_mode):
 700                                                 writemsg_level(
 701                                                         _("!!! Unable to fetch file since "
 702                                                         "a directory is in the way: \n"
 703                                                         "!!!   %s\n") % myfile_path,
 704                                                         level=logging.ERROR, noiselevel=-1)
 705                                                 return 0
 706
 707                                         if mystat.st_size == 0:
 708                                                 if distdir_writable:
 709                                                         try:
 710                                                                 os.unlink(myfile_path)
 711                                                         except OSError:
 712                                                                 pass
 713                                         elif distdir_writable:
 714                                                 if mystat.st_size < fetch_resume_size and \
 715                                                         mystat.st_size < size:
 716                                                         # If the file already exists and the size does not
 717                                                         # match the existing digests, it may be that the
 718                                                         # user is attempting to update the digest. In this
 719                                                         # case, the digestgen() function will advise the
 720                                                         # user to use `ebuild --force foo.ebuild manifest`
 721                                                         # in order to force the old digests to be replaced.
 722                                                         # Since the user may want to keep this file, rename
 723                                                         # it instead of deleting it.
 724                                                         writemsg(_(">>> Renaming distfile with size "
 725                                                                 "%d (smaller than " "PORTAGE_FETCH_RESU"
 726                                                                 "ME_MIN_SIZE)\n") % mystat.st_size)
 727                                                         temp_filename = \
 728                                                                 _checksum_failure_temp_file(
 729                                                                 mysettings["DISTDIR"], myfile)
 730                                                         writemsg_stdout(_("Refetching... "
 731                                                                 "File renamed to '%s'\n\n") % \
 732                                                                 temp_filename, noiselevel=-1)
 733                                                 elif mystat.st_size >= size:
 734                                                         temp_filename = \
 735                                                                 _checksum_failure_temp_file(
 736                                                                 mysettings["DISTDIR"], myfile)
 737                                                         writemsg_stdout(_("Refetching... "
 738                                                                 "File renamed to '%s'\n\n") % \
 739                                                                 temp_filename, noiselevel=-1)
 740
 741                                 if distdir_writable and ro_distdirs:
 742                                         readonly_file = None
 743                                         for x in ro_distdirs:
 744                                                 filename = os.path.join(x, myfile)
 745                                                 match, mystat = _check_distfile(
 746                                                         filename, pruned_digests, eout, hash_filter=hash_filter)
 747                                                 if match:
 748                                                         readonly_file = filename
 749                                                         break
 750                                         if readonly_file is not None:
 751                                                 try:
 752                                                         os.unlink(myfile_path)
 753                                                 except OSError as e:
 754                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 755                                                                 raise
 756                                                         del e
 757                                                 os.symlink(readonly_file, myfile_path)
 758                                                 continue
 759
 760                                 # this message is shown only after we know that
 761                                 # the file is not already fetched
 762                                 if not has_space:
 763                                         writemsg(_("!!! Insufficient space to store %s in %s\n") % \
 764                                                 (myfile, mysettings["DISTDIR"]), noiselevel=-1)
 765
 766                                         if has_space_superuser:
 767                                                 writemsg(_("!!! Insufficient privileges to use "
 768                                                         "remaining space.\n"), noiselevel=-1)
 769                                                 if userfetch:
 770                                                         writemsg(_("!!! You may set FEATURES=\"-userfetch\""
 771                                                                 " in /etc/portage/make.conf in order to fetch with\n"
 772                                                                 "!!! superuser privileges.\n"), noiselevel=-1)
 773
 774                                 if fsmirrors and not os.path.exists(myfile_path) and has_space:
 775                                         for mydir in fsmirrors:
 776                                                 mirror_file = os.path.join(mydir, myfile)
 777                                                 try:
 778                                                         shutil.copyfile(mirror_file, myfile_path)
 779                                                         writemsg(_("Local mirror has file: %s\n") % myfile)
 780                                                         break
 781                                                 except (IOError, OSError) as e:
 782                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 783                                                                 raise
 784                                                         del e
 785
 786                                 try:
 787                                         mystat = os.stat(myfile_path)
 788                                 except OSError as e:
 789                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 790                                                 raise
 791                                         del e
 792                                 else:
 793                                         # Skip permission adjustment for symlinks, since we don't
 794                                         # want to modify anything outside of the primary DISTDIR,
 795                                         # and symlinks typically point to PORTAGE_RO_DISTDIRS.
 796                                         if not os.path.islink(myfile_path):
 797                                                 try:
 798                                                         apply_secpass_permissions(myfile_path,
 799                                                                 gid=portage_gid, mode=0o664, mask=0o2,
 800                                                                 stat_cached=mystat)
 801                                                 except PortageException as e:
 802                                                         if not os.access(myfile_path, os.R_OK):
 803                                                                 writemsg(_("!!! Failed to adjust permissions:"
 804                                                                         " %s\n") % (e,), noiselevel=-1)
 805
 806                                         # If the file is empty then it's obviously invalid. Remove
 807                                         # the empty file and try to download if possible.
 808                                         if mystat.st_size == 0:
 809                                                 if distdir_writable:
 810                                                         try:
 811                                                                 os.unlink(myfile_path)
 812                                                         except EnvironmentError:
 813                                                                 pass
 814                                         elif myfile not in mydigests:
 815                                                 # We don't have a digest, but the file exists.  We must
 816                                                 # assume that it is fully downloaded.
 817                                                 continue
 818                                         else:
 819                                                 if mystat.st_size < mydigests[myfile]["size"] and \
 820                                                         not restrict_fetch:
 821                                                         fetched = 1 # Try to resume this download.
 822                                                 elif parallel_fetchonly and \
 823                                                         mystat.st_size == mydigests[myfile]["size"]:
 824                                                         eout = EOutput()
 825                                                         eout.quiet = \
 826                                                                 mysettings.get("PORTAGE_QUIET") == "1"
 827                                                         eout.ebegin(
 828                                                                 "%s size ;-)" % (myfile, ))
 829                                                         eout.eend(0)
 830                                                         continue
 831                                                 else:
 832                                                         digests = _filter_unaccelarated_hashes(mydigests[myfile])
 833                                                         if hash_filter is not None:
 834                                                                 digests = _apply_hash_filter(digests, hash_filter)
 835                                                         verified_ok, reason = verify_all(myfile_path, digests)
 836                                                         if not verified_ok:
 837                                                                 writemsg(_("!!! Previously fetched"
 838                                                                         " file: '%s'\n") % myfile, noiselevel=-1)
 839                                                                 writemsg(_("!!! Reason: %s\n") % reason[0],
 840                                                                         noiselevel=-1)
 841                                                                 writemsg(_("!!! Got:      %s\n"
 842                                                                         "!!! Expected: %s\n") % \
 843                                                                         (reason[1], reason[2]), noiselevel=-1)
 844                                                                 if reason[0] == _("Insufficient data for checksum verification"):
 845                                                                         return 0
 846                                                                 if distdir_writable:
 847                                                                         temp_filename = \
 848                                                                                 _checksum_failure_temp_file(
 849                                                                                 mysettings["DISTDIR"], myfile)
 850                                                                         writemsg_stdout(_("Refetching... "
 851                                                                                 "File renamed to '%s'\n\n") % \
 852                                                                                 temp_filename, noiselevel=-1)
 853                                                         else:
 854                                                                 eout = EOutput()
 855                                                                 eout.quiet = \
 856                                                                         mysettings.get("PORTAGE_QUIET", None) == "1"
 857                                                                 if digests:
 858                                                                         digests = list(digests)
 859                                                                         digests.sort()
 860                                                                         eout.ebegin(
 861                                                                                 "%s %s ;-)" % (myfile, " ".join(digests)))
 862                                                                         eout.eend(0)
 863                                                                 continue # fetch any remaining files
 864
 865                         # Create a reversed list since that is optimal for list.pop().
 866                         uri_list = filedict[myfile][:]
 867                         uri_list.reverse()
 868                         checksum_failure_count = 0
 869                         tried_locations = set()
 870                         while uri_list:
 871                                 loc = uri_list.pop()
 872                                 # Eliminate duplicates here in case we've switched to
 873                                 # "primaryuri" mode on the fly due to a checksum failure.
 874                                 if loc in tried_locations:
 875                                         continue
 876                                 tried_locations.add(loc)
 877                                 if listonly:
 878                                         writemsg_stdout(loc+" ", noiselevel=-1)
 879                                         continue
 880                                 # allow different fetchcommands per protocol
 881                                 protocol = loc[0:loc.find("://")]
 882
 883                                 global_config_path = GLOBAL_CONFIG_PATH
 884                                 if portage.const.EPREFIX:
 885                                         global_config_path = os.path.join(portage.const.EPREFIX,
 886                                                         GLOBAL_CONFIG_PATH.lstrip(os.sep))
 887
 888                                 missing_file_param = False
 889                                 fetchcommand_var = "FETCHCOMMAND_" + protocol.upper()
 890                                 fetchcommand = mysettings.get(fetchcommand_var)
 891                                 if fetchcommand is None:
 892                                         fetchcommand_var = "FETCHCOMMAND"
 893                                         fetchcommand = mysettings.get(fetchcommand_var)
 894                                         if fetchcommand is None:
 895                                                 writemsg_level(
 896                                                         _("!!! %s is unset. It should "
 897                                                         "have been defined in\n!!! %s/make.globals.\n") \
 898                                                         % (fetchcommand_var, global_config_path),
 899                                                         level=logging.ERROR, noiselevel=-1)
 900                                                 return 0
 901                                 if "${FILE}" not in fetchcommand:
 902                                         writemsg_level(
 903                                                 _("!!! %s does not contain the required ${FILE}"
 904                                                 " parameter.\n") % fetchcommand_var,
 905                                                 level=logging.ERROR, noiselevel=-1)
 906                                         missing_file_param = True
 907
 908                                 resumecommand_var = "RESUMECOMMAND_" + protocol.upper()
 909                                 resumecommand = mysettings.get(resumecommand_var)
 910                                 if resumecommand is None:
 911                                         resumecommand_var = "RESUMECOMMAND"
 912                                         resumecommand = mysettings.get(resumecommand_var)
 913                                         if resumecommand is None:
 914                                                 writemsg_level(
 915                                                         _("!!! %s is unset. It should "
 916                                                         "have been defined in\n!!! %s/make.globals.\n") \
 917                                                         % (resumecommand_var, global_config_path),
 918                                                         level=logging.ERROR, noiselevel=-1)
 919                                                 return 0
 920                                 if "${FILE}" not in resumecommand:
 921                                         writemsg_level(
 922                                                 _("!!! %s does not contain the required ${FILE}"
 923                                                 " parameter.\n") % resumecommand_var,
 924                                                 level=logging.ERROR, noiselevel=-1)
 925                                         missing_file_param = True
 926
 927                                 if missing_file_param:
 928                                         writemsg_level(
 929                                                 _("!!! Refer to the make.conf(5) man page for "
 930                                                 "information about how to\n!!! correctly specify "
 931                                                 "FETCHCOMMAND and RESUMECOMMAND.\n"),
 932                                                 level=logging.ERROR, noiselevel=-1)
 933                                         if myfile != os.path.basename(loc):
 934                                                 return 0
 935
 936                                 if not can_fetch:
 937                                         if fetched != 2:
 938                                                 try:
 939                                                         mysize = os.stat(myfile_path).st_size
 940                                                 except OSError as e:
 941                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 942                                                                 raise
 943                                                         del e
 944                                                         mysize = 0
 945
 946                                                 if mysize == 0:
 947                                                         writemsg(_("!!! File %s isn't fetched but unable to get it.\n") % myfile,
 948                                                                 noiselevel=-1)
 949                                                 elif size is None or size > mysize:
 950                                                         writemsg(_("!!! File %s isn't fully fetched, but unable to complete it\n") % myfile,
 951                                                                 noiselevel=-1)
 952                                                 else:
 953                                                         writemsg(_("!!! File %s is incorrect size, "
 954                                                                 "but unable to retry.\n") % myfile, noiselevel=-1)
 955                                                 return 0
 956                                         else:
 957                                                 continue
 958
 959                                 if fetched != 2 and has_space:
 960                                         #we either need to resume or start the download
 961                                         if fetched == 1:
 962                                                 try:
 963                                                         mystat = os.stat(myfile_path)
 964                                                 except OSError as e:
 965                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 966                                                                 raise
 967                                                         del e
 968                                                         fetched = 0
 969                                                 else:
 970                                                         if mystat.st_size < fetch_resume_size:
 971                                                                 writemsg(_(">>> Deleting distfile with size "
 972                                                                         "%d (smaller than " "PORTAGE_FETCH_RESU"
 973                                                                         "ME_MIN_SIZE)\n") % mystat.st_size)
 974                                                                 try:
 975                                                                         os.unlink(myfile_path)
 976                                                                 except OSError as e:
 977                                                                         if e.errno not in \
 978                                                                                 (errno.ENOENT, errno.ESTALE):
 979                                                                                 raise
 980                                                                         del e
 981                                                                 fetched = 0
 982                                         if fetched == 1:
 983                                                 #resume mode:
 984                                                 writemsg(_(">>> Resuming download...\n"))
 985                                                 locfetch=resumecommand
 986                                                 command_var = resumecommand_var
 987                                         else:
 988                                                 #normal mode:
 989                                                 locfetch=fetchcommand
 990                                                 command_var = fetchcommand_var
 991                                         writemsg_stdout(_(">>> Downloading '%s'\n") % \
 992                                                 _hide_url_passwd(loc))
 993                                         variables = {
 994                                                 "URI":     loc,
 995                                                 "FILE":    myfile
 996                                         }
 997
 998                                         for k in ("DISTDIR", "PORTAGE_SSH_OPTS"):
 999                                                 try:
1000                                                         variables[k] = mysettings[k]
1001                                                 except KeyError:
1002                                                         pass
1003
1004                                         myfetch = shlex_split(locfetch)
1005                                         myfetch = [varexpand(x, mydict=variables) for x in myfetch]
1006                                         myret = -1
1007                                         try:
1008
1009                                                 myret = _spawn_fetch(mysettings, myfetch)
1010
1011                                         finally:
1012                                                 try:
1013                                                         apply_secpass_permissions(myfile_path,
1014                                                                 gid=portage_gid, mode=0o664, mask=0o2)
1015                                                 except FileNotFound:
1016                                                         pass
1017                                                 except PortageException as e:
1018                                                         if not os.access(myfile_path, os.R_OK):
1019                                                                 writemsg(_("!!! Failed to adjust permissions:"
1020                                                                         " %s\n") % str(e), noiselevel=-1)
1021                                                         del e
1022
1023                                         # If the file is empty then it's obviously invalid.  Don't
1024                                         # trust the return value from the fetcher.  Remove the
1025                                         # empty file and try to download again.
1026                                         try:
1027                                                 if os.stat(myfile_path).st_size == 0:
1028                                                         os.unlink(myfile_path)
1029                                                         fetched = 0
1030                                                         continue
1031                                         except EnvironmentError:
1032                                                 pass
1033
1034                                         if mydigests is not None and myfile in mydigests:
1035                                                 try:
1036                                                         mystat = os.stat(myfile_path)
1037                                                 except OSError as e:
1038                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
1039                                                                 raise
1040                                                         del e
1041                                                         fetched = 0
1042                                                 else:
1043
1044                                                         if stat.S_ISDIR(mystat.st_mode):
1045                                                                 # This can happen if FETCHCOMMAND erroneously
1046                                                                 # contains wget's -P option where it should
1047                                                                 # instead have -O.
1048                                                                 writemsg_level(
1049                                                                         _("!!! The command specified in the "
1050                                                                         "%s variable appears to have\n!!! "
1051                                                                         "created a directory instead of a "
1052                                                                         "normal file.\n") % command_var,
1053                                                                         level=logging.ERROR, noiselevel=-1)
1054                                                                 writemsg_level(
1055                                                                         _("!!! Refer to the make.conf(5) "
1056                                                                         "man page for information about how "
1057                                                                         "to\n!!! correctly specify "
1058                                                                         "FETCHCOMMAND and RESUMECOMMAND.\n"),
1059                                                                         level=logging.ERROR, noiselevel=-1)
1060                                                                 return 0
1061
1062                                                         # no exception?  file exists. let digestcheck() report
1063                                                         # an appropriately for size or checksum errors
1064
1065                                                         # If the fetcher reported success and the file is
1066                                                         # too small, it's probably because the digest is
1067                                                         # bad (upstream changed the distfile).  In this
1068                                                         # case we don't want to attempt to resume. Show a
1069                                                         # digest verification failure to that the user gets
1070                                                         # a clue about what just happened.
1071                                                         if myret != os.EX_OK and \
1072                                                                 mystat.st_size < mydigests[myfile]["size"]:
1073                                                                 # Fetch failed... Try the next one... Kill 404 files though.
1074                                                                 if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
1075                                                                         html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M)
1076                                                                         with io.open(
1077                                                                                 _unicode_encode(myfile_path,
1078                                                                                 encoding=_encodings['fs'], errors='strict'),
1079                                                                                 mode='r', encoding=_encodings['content'], errors='replace'
1080                                                                                 ) as f:
1081                                                                                 if html404.search(f.read()):
1082                                                                                         try:
1083                                                                                                 os.unlink(mysettings["DISTDIR"]+"/"+myfile)
1084                                                                                                 writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n"))
1085                                                                                                 fetched = 0
1086                                                                                                 continue
1087                                                                                         except (IOError, OSError):
1088                                                                                                 pass
1089                                                                 fetched = 1
1090                                                                 continue
1091                                                         if True:
1092                                                                 # File is the correct size--check the checksums for the fetched
1093                                                                 # file NOW, for those users who don't have a stable/continuous
1094                                                                 # net connection. This way we have a chance to try to download
1095                                                                 # from another mirror...
1096                                                                 digests = _filter_unaccelarated_hashes(mydigests[myfile])
1097                                                                 if hash_filter is not None:
1098                                                                         digests = _apply_hash_filter(digests, hash_filter)
1099                                                                 verified_ok, reason = verify_all(myfile_path, digests)
1100                                                                 if not verified_ok:
1101                                                                         writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
1102                                                                                 noiselevel=-1)
1103                                                                         writemsg(_("!!! Reason: %s\n") % reason[0],
1104                                                                                 noiselevel=-1)
1105                                                                         writemsg(_("!!! Got:      %s\n!!! Expected: %s\n") % \
1106                                                                                 (reason[1], reason[2]), noiselevel=-1)
1107                                                                         if reason[0] == _("Insufficient data for checksum verification"):
1108                                                                                 return 0
1109                                                                         temp_filename = \
1110                                                                                 _checksum_failure_temp_file(
1111                                                                                 mysettings["DISTDIR"], myfile)
1112                                                                         writemsg_stdout(_("Refetching... "
1113                                                                                 "File renamed to '%s'\n\n") % \
1114                                                                                 temp_filename, noiselevel=-1)
1115                                                                         fetched=0
1116                                                                         checksum_failure_count += 1
1117                                                                         if checksum_failure_count == \
1118                                                                                 checksum_failure_primaryuri:
1119                                                                                 # Switch to "primaryuri" mode in order
1120                                                                                 # to increase the probablility of
1121                                                                                 # of success.
1122                                                                                 primaryuris = \
1123                                                                                         primaryuri_dict.get(myfile)
1124                                                                                 if primaryuris:
1125                                                                                         uri_list.extend(
1126                                                                                                 reversed(primaryuris))
1127                                                                         if checksum_failure_count >= \
1128                                                                                 checksum_failure_max_tries:
1129                                                                                 break
1130                                                                 else:
1131                                                                         eout = EOutput()
1132                                                                         eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
1133                                                                         if digests:
1134                                                                                 eout.ebegin("%s %s ;-)" % \
1135                                                                                         (myfile, " ".join(sorted(digests))))
1136                                                                                 eout.eend(0)
1137                                                                         fetched=2
1138                                                                         break
1139                                         else:
1140                                                 if not myret:
1141                                                         fetched=2
1142                                                         break
1143                                                 elif mydigests!=None:
1144                                                         writemsg(_("No digest file available and download failed.\n\n"),
1145                                                                 noiselevel=-1)
1146                 finally:
1147                         if use_locks and file_lock:
1148                                 unlockfile(file_lock)
1149                                 file_lock = None
1150
1151                 if listonly:
1152                         writemsg_stdout("\n", noiselevel=-1)
1153                 if fetched != 2:
1154                         if restrict_fetch and not restrict_fetch_msg:
1155                                 restrict_fetch_msg = True
1156                                 msg = _("\n!!! %s/%s"
1157                                         " has fetch restriction turned on.\n"
1158                                         "!!! This probably means that this "
1159                                         "ebuild's files must be downloaded\n"
1160                                         "!!! manually.  See the comments in"
1161                                         " the ebuild for more information.\n\n") % \
1162                                         (mysettings["CATEGORY"], mysettings["PF"])
1163                                 writemsg_level(msg,
1164                                         level=logging.ERROR, noiselevel=-1)
1165                         elif restrict_fetch:
1166                                 pass
1167                         elif listonly:
1168                                 pass
1169                         elif not filedict[myfile]:
1170                                 writemsg(_("Warning: No mirrors available for file"
1171                                         " '%s'\n") % (myfile), noiselevel=-1)
1172                         else:
1173                                 writemsg(_("!!! Couldn't download '%s'. Aborting.\n") % myfile,
1174                                         noiselevel=-1)
1175
1176                         if listonly:
1177                                 failed_files.add(myfile)
1178                                 continue
1179                         elif fetchonly:
1180                                 failed_files.add(myfile)
1181                                 continue
1182                         return 0
1183         if failed_files:
1184                 return 0
1185         return 1